#+TITLE: Alya Perf #+AUTHOR: Borrell, Schnorr, Legrand #+LATEX_HEADER: \usepackage[margin=2cm,a4paper]{geometry} #+STARTUP: overview indent #+TAGS: noexport(n) deprecated(d) #+EXPORT_SELECT_TAGS: export #+EXPORT_EXCLUDE_TAGS: noexport #+SEQ_TODO: TODO(t!) STARTED(s!) WAITING(w!) | DONE(d!) CANCELLED(c!) DEFERRED(f!) This document is in English. * Installation ** G5K Use Lucas's buster-v13 g5k image. *** ScoreP 4.0 Download at: http://www.score-p.org/ Extract, then: #+begin_src shell :results output wget http://www.vi-hps.org/upload/packages/scorep/scorep-4.0.tar.gz tar xfz scorep-4.0.tar.gz cd scorep-4.0 ./configure --prefix=$HOME/install/scorep-4.0/ make -j 48 make install #+end_src *** Alya Installation **** SFC branch #+begin_src shell :results output svn checkout svn+ssh://bsc21835@dt01.bsc.es/gpfs/projects/bsc21/svnroot/Alya/branches/sfc alya-bsc-sfc #+end_src Most of the G5K experiments were carried out using this revision: #+BEGIN_EXAMPLE Working Copy Root Path: /home/schnorr/misc/alya-bsc-sfc URL: svn+ssh://bsc21835@dt01.bsc.es/gpfs/projects/bsc21/svnroot/Alya/branches/sfc Relative URL: ^/branches/sfc Repository Root: svn+ssh://bsc21835@dt01.bsc.es/gpfs/projects/bsc21/svnroot/Alya Repository UUID: 2cd113d9-5f18-0410-b867-96f721f0e0d9 Revision: 8538 Node Kind: directory Schedule: normal Last Changed Author: bsc21665 Last Changed Rev: 8495 Last Changed Date: 2018-02-01 15:54:04 -0200 (Thu, 01 Feb 2018) #+END_EXAMPLE **** Our Instrumentation with Score-P The instrumentation should look like this: #+begin_example Index: Sources/kernel/master/Alya.f90 =================================================================== --- Sources/kernel/master/Alya.f90 (revision 8442) +++ Sources/kernel/master/Alya.f90 (working copy) @@ -1,3 +1,4 @@ +#include "scorep/SCOREP_User.inc" !> @file Alya.f90 !! @author Guillaume Houzeaux !! @brief Ayla main @@ -20,7 +21,13 @@ use def_master, only : kfl_goblk use def_master, only : kfl_gocou use def_coupli, only : kfl_gozon + use mod_parall, only : PAR_MY_WORLD_RANK implicit none + INTEGER :: iter,ierror + character*100 striter + character*100 iterfile + real :: tnow + real, dimension(2) :: tarray ! ! DLB should be disabled as we only wabnt to activate it for particular loops ! Master does not disble to lend its resources automatically @@ -39,6 +46,10 @@ call Parall(22270_ip) + write(iterfile,'(a,i4.4,a)') 'iterations-', PAR_MY_WORLD_RANK, '.csv' + open(unit=2, file=iterfile) + + iter = 1 optimization: do while ( kfl_goopt == 1 ) call Iniunk() @@ -49,6 +60,11 @@ time: do while ( kfl_gotim == 1 ) call Timste() + call ETIME(tarray, tnow) + write(2,*) tnow, tarray(1), tarray(2), PAR_MY_WORLD_RANK, iter + + write(striter, '(a,i3.3)') 'iter',iter + SCOREP_USER_REGION_BY_NAME_BEGIN(striter, SCOREP_USER_REGION_TYPE_PHASE) reset: do call Begste() @@ -77,6 +93,13 @@ call Endste() + SCOREP_USER_REGION_BY_NAME_END(striter) + call ETIME(tarray, tnow) + write(2,*) tnow, tarray(1), tarray(2), PAR_MY_WORLD_RANK, iter + iter = iter + 1 + + + call Filter(ITASK_ENDTIM) call Output(ITASK_ENDTIM) @@ -91,6 +114,9 @@ end do optimization + close(2) + call Turnof() + end program Alya Index: Sources/services/parall/par_prepro.f90 =================================================================== --- Sources/services/parall/par_prepro.f90 (revision 9154) +++ Sources/services/parall/par_prepro.f90 (working copy) @@ -23,6 +23,7 @@ use mod_redistribute, only : redistribute use mod_redistribute, only : gather_to_master use mod_parall, only : PAR_WORLD_SIZE + use mod_parall, only : PAR_MY_WORLD_RANK use mod_parall, only : PAR_METIS4 use mod_parall, only : PAR_SFC use mod_parall, only : PAR_ORIENTED_BIN @@ -39,6 +40,13 @@ integer(ip) :: npoin_tmp, nelem_tmp, nboun_tmp,ndims real(rp) :: time1,time2,time3,time4,time5 character(100) :: messa_integ + ! LUCAS + character*100 dfile + integer(ip) ii + ! END LUCAS + + + ! ! Intermediate variables for partitioning ! @@ -384,4 +392,19 @@ end if + ! LUCAS + write(dfile,'(a,i4.4,a)') 'domain-', PAR_MY_WORLD_RANK, '.csv' + open(unit=12345, file=dfile) + + write(12345,*) "T1", PAR_MY_WORLD_RANK, nelem, npoin, nboun, npoi3-npoi2 + do ii = 1, nelem + write(12345,*) "T2", PAR_MY_WORLD_RANK, ii, ltype(ii) + end do + do ii = 1, nboun + write(12345,*) "T3", PAR_MY_WORLD_RANK, ii, ltypb(ii) + end do + + close(12345) + ! END LUCAS + end subroutine par_prepro #+end_example **** Compilation In the =./ThirdParties/metis-4.0/=, compile Metis: #+begin_src shell :results output cd ./ThirdParties/metis-4.0/ make #+end_src In the =./Executables/unix/= Alya directory, do. Copy =./configure.in/config_gfortran.in= to config.in. Make it look like this (in the begginning): #+BEGIN_EXAMPLE SCOREP = ~/install/scorep-4.0/bin/scorep MPIF90 = ~//spack-ALYA/opt/spack/linux-debian9-x86_64/gcc-6.3.0/openmpi-3.0.0-a7g33v4ulwtb4g2verliyelvtifybrq3/bin/mpif90 MPICC = ~//spack-ALYA/opt/spack/linux-debian9-x86_64/gcc-6.3.0/openmpi-3.0.0-a7g33v4ulwtb4g2verliyelvtifybrq3/bin/mpicc F77 = $(SCOREP) --user --nocompiler --nopomp --noopenmp $(MPIF90) -cpp F90 = $(SCOREP) --user --nocompiler --nopomp --noopenmp $(MPIF90) -cpp FCOCC = $(SCOREP) --user --nocompiler --nopomp --noopenmp $(MPICC) -c #+END_EXAMPLE Then, configure the compilation #+begin_src shell :results output ./configure nastin parall make -j 48 #+end_src The compilation fails with #+BEGIN_EXAMPLE /gpfs/home/bsc21/bsc21835/alya-bsc-sfc/Sources/kernel/coupli/mod_commdom_alya.f90:489:2: CPLNG%sendrecv(1,6) = (current_task==ITASK_BEGSTE).and.( current_when==ITASK_BEFORE).and.& ! \ 1 Error: Unclassifiable statement at (1) #+END_EXAMPLE The solution is to comment those lines (and =make= again). The compilation fails with =par_prepro.f90=, just comment (or remove) the line that has =use mod_par_partit_sfc,=. *** R Installed in the image. ** Jequi * Scripts ** Get info from the platform *** Get machine information before doing the experiment #+begin_src sh :results output :tangle scripts/get_info.sh :tangle-mode (identity #o755) #!/bin/bash # Script for to get machine information before doing the experiment set +e # Don't fail fast since some information is maybe not available title="Experiment results" starpu_build="" inputfile="" host="$(hostname | sed 's/[0-9]*//g' | cut -d'.' -f1)" help_script() { cat << EOF Usage: $0 [options] outputfile.org Script for to get machine information before doing the experiment OPTIONS: -h Show this message -t Title of the output file -s Path to the StarPU installation -i Input file name if doing SimGrid simulation based on input EOF } # Parsing options while getopts "t:s:i:h" opt; do case $opt in t) title="$OPTARG" ;; s) starpu_build="$OPTARG" ;; i) inputfile="$OPTARG" ;; h) help_script exit 4 ;; \?) echo "Invalid option: -$OPTARG" help_script exit 3 ;; esac done shift $((OPTIND - 1)) filedat=$1 if [[ $# != 1 ]]; then echo 'ERROR!' help_script exit 2 fi ################################################## # Preambule of the output file echo "#+TITLE: $title" >> $filedat echo "#+DATE: $(eval date)" >> $filedat echo "#+AUTHOR: $(eval whoami)" >> $filedat echo "#+MACHINE: $(eval hostname)" >> $filedat echo "#+FILE: $(eval basename $filedat)" >> $filedat if [[ -n "$inputfile" ]]; then echo "#+INPUTFILE: $inputfile" >> $filedat fi echo " " >> $filedat ################################################## # Collecting metadata echo "* MACHINE INFO:" >> $filedat echo "** PEOPLE LOGGED WHEN EXPERIMENT STARTED:" >> $filedat who >> $filedat echo "############################################" >> $filedat echo "** ENVIRONMENT VARIABLES:" >> $filedat env >> $filedat echo "############################################" >> $filedat echo "** HOSTNAME:" >> $filedat hostname >> $filedat echo "############################################" >> $filedat if [[ -n $(command -v lstopo) ]]; then echo "** MEMORY HIERARCHY:" >> $filedat lstopo --of console >> $filedat echo "############################################" >> $filedat fi if [[ -n "$starpu_build" ]]; then echo "** STARPU MACHINE DISPLAY:" >> $filedat $starpu_build/bin/starpu_machine_display 1> tmp 2> /dev/null cat tmp >> $filedat rm -f tmp echo "############################################" >> $filedat fi if [ -f /proc/cpuinfo ]; then echo "** CPU INFO:" >> $filedat cat /proc/cpuinfo >> $filedat echo "############################################" >> $filedat fi if [ -f /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then echo "** CPU GOVERNOR:" >> $filedat ONLINECPUS=$(for CPU in $(find /sys/devices/system/cpu/ | grep cpu[0-9]*$); do [[ $(cat $CPU/online) -eq 1 ]] && echo $CPU; done | grep cpu[0-9]*$ | sed 's/.*cpu//') for PU in ${ONLINECPUS}; do echo -n "CPU frequency for cpu${PU}: " >> $filedat cat /sys/devices/system/cpu/cpu${PU}/cpufreq/scaling_governor >> $filedat done echo "############################################" >> $filedat fi if [ -f /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq ]; then echo "** CPU FREQUENCY:" >> $filedat ONLINECPUS=$(for CPU in $(find /sys/devices/system/cpu/ | grep cpu[0-9]*$); do [[ $(cat $CPU/online) -eq 1 ]] && echo $CPU; done | grep cpu[0-9]*$ | sed 's/.*cpu//') for PU in ${ONLINECPUS}; do echo -n "CPU frequency for cpu${PU}: " >> $filedat cat /sys/devices/system/cpu/cpu${PU}/cpufreq/scaling_cur_freq >> $filedat done echo "############################################" >> $filedat fi if [ -f /usr/bin/cpufreq-info ]; then echo "** CPUFREQ_INFO" >> $filedat cpufreq-info >> $filedat echo "############################################" >> $filedat fi if [ -f /usr/bin/lspci ]; then echo "** LSPCI" >> $filedat lspci >> $filedat echo "############################################" >> $filedat fi if [ -f /usr/bin/ompi_info ]; then echo "** OMPI_INFO" >> $filedat ompi_info --all >> $filedat echo "############################################" >> $filedat fi if [ -f /sbin/ifconfig ]; then echo "** IFCONFIG" >> $filedat /sbin/ifconfig >> $filedat echo "############################################" >> $filedat fi if [[ -n $(command -v nvidia-smi) ]]; then echo "** GPU INFO FROM NVIDIA-SMI:" >> $filedat nvidia-smi -q >> $filedat echo "############################################" >> $filedat fi if [ -f /proc/version ]; then echo "** LINUX AND GCC VERSIONS:" >> $filedat cat /proc/version >> $filedat echo "############################################" >> $filedat fi if [[ -n $(command -v module) ]]; then echo "** MODULES:" >> $filedat module list 2>> $filedat echo "############################################" >> $filedat fi #+end_src ** Common Scripts *** Colors + error report #+begin_src sh :results output :tangle scripts/colors_error_report.sh :tangle-mode (identity #o755) #!/bin/bash set -euo pipefail # Error function function error_script { ERRORMSG=${1:-} echo "Error: $ERRORMSG" exit 1 } # Check functions function check_var { VAR=${1:-} MSG=${2:-} if [[ -z "$VAR" ]]; then error_script $MSG fi } function check_bin { BIN=${1:-} MSG=${2:-} if [[ ! -x "$BIN" ]]; then error_script $MSG fi } function check_env { ENV=${1:-} MSG=${2:-} if [ -z ${ENV+x} ]; then error_script $MSG fi } function info { INFOMSG=${1:-} echo "Info: $INFOMSG" } #+end_src *** Create experimental directory (only functions) #+begin_src sh :results output :tangle scripts/create_experiment_dir.sh :tangle-mode (identity #o755) #!/bin/bash # fail often, so we are robust set -euo pipefail # source functions DIR=$(dirname $0) source $DIR/colors_error_report.sh # Function to cat a file within an example block function org_example_file { FILE=${1:-} if [ -e "$FILE" ]; then echo "#+BEGIN_EXAMPLE" cat $FILE echo "#+END_EXAMPLE" fi } # Function that returns the unique org filename given an experiment unique identifier function org_err_file { UNIQUE=${1:-} if [ -z "$UNIQUE" ]; then error_script "Error: Unique key not passed as parameter to org_err_file." fi echo $(basename $UNIQUE .org)_err.org } # Function that returns the unique org filename given an experiment unique identifier function org_file { UNIQUE=${1:-} if [ -z "$UNIQUE" ]; then error_script "Error: Unique key not passed as parameter to org_file." fi echo $(basename $UNIQUE .org).org } function org_info_file { UNIQUE=${1:-} if [ -z "$UNIQUE" ]; then error_script "Error: Unique key not passed as parameter to org_file." fi echo $(basename $UNIQUE .org)_get_info.org } # Function that returns the unique directory name given an experiment unique identifier function dir_file { UNIQUE=${1:-} if [ -z "$UNIQUE" ]; then error_script "Error: Unique key not passed as parameter to dir_file." fi echo $(basename $UNIQUE .org).dir } function create_experiment_dir { UNIQUE=${1:-} if [ -z "$UNIQUE" ]; then error_script "Error: Unique key not passed as parameter to create_experiment_directory." fi # Preparation of the output (org and dir) OUTPUTORG=$(org_file $UNIQUE) OUTPUTORGDIR=$(dir_file $UNIQUE) if [ -e $OUTPUTORG ]; then error_script "$OUTPUTORG already exists. Please, remove it or use another unique name." usage; exit; else info "Experiment Org file: $OUTPUTORG" fi if [ -d $OUTPUTORGDIR ]; then error_script "$OUTPUTORGDIR directory already exists. Please, remove it or use another unique name." usage; exit; else info "Experiment Directory: $OUTPUTORGDIR" fi # Everything seems okay, let's prepare the terrain touch $OUTPUTORG if [ $? -ne 0 ]; then error_script "Creation of $OUTPUTORG did not work." exit; fi mkdir $OUTPUTORGDIR if [ $? -ne 0 ]; then error_script "Creation of $OUTPUTORGDIR did not work." exit; fi } #+end_src *** Detect Hyperthreading Inspired by: http://unix.stackexchange.com/questions/33450/checking-if-hyperthreading-is-enabled-or-not/33509#33509 #+begin_src R :results output :session :exports both :tangle scripts/hyperthreading.sh :tangle-mode (identity #o755) #!/bin/bash CPUFILE=/proc/cpuinfo test -f $CPUFILE || exit 1 NUMPHYCPU=`grep "physical id" $CPUFILE | sort -u | wc -l` NUMLOGCORE=`grep "processor" $CPUFILE | wc -l` NUMPHYCORE=`grep "core id" $CPUFILE | sort -u | wc -l` TOTALNUMPHYCORE=$(echo "$NUMPHYCPU * $NUMPHYCORE" | bc) MODEL=`grep "model name" $CPUFILE | sort -u | cut -d : -f 2- | sed "s/^[[:space:]]*//"` echo "This system has $NUMPHYCPU CPUs, of model \"$MODEL\"." echo "Each physical CPU is equipped with $NUMPHYCORE physical cores (total is $TOTALNUMPHYCORE)." if [ $TOTALNUMPHYCORE -ne $NUMLOGCORE ]; then echo "Hyperthreading is ON. So, there are $NUMLOGCORE logical cores." else echo "Hyperthreading is OFF." fi exit #+end_src *** Disable Hyperthreading #+begin_src sh :results output :tangle scripts/disable_hyperthreading.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) #First, enable all cores for PU in `find /sys/devices/system/cpu/ |grep cpu[0-9]*$`; do echo "Enabling $PU now." sudo zsh -c "echo 1 > ${PU}/online" done HYPERTHREADING=`$DIR/hyperthreading.sh | grep -e "Hyperthreading is ON" | wc -l` if [ $HYPERTHREADING -eq 0 ]; then echo "Hyperthreading is OFF, so disabling is not necessary." exit else echo "Hyperthreading is ON." fi echo "The number of PUs now is $(hwloc-ls --only PU | wc -l)." echo "I will disable hyperthreading now." # Disable hyperthreading # Only run this if you are sure # - Hyperthreading is enabled # - Each physical core has two processing units (PU) # - hwloc-ls is installed and reports two PU per physical core for PU in `hwloc-ls --only PU | cat -n | grep -e "[[:space:]]*[0-9]*[02468][[:space:]]*PU" | sed -e "s/^[^(]*(P#\\([0-9]*\))/\1/"`; do echo "Disabling PU $PU now." sudo zsh -c "echo 0 > /sys/devices/system/cpu/cpu${PU}/online" done echo "The number of PUs now is $(hwloc-ls --only PU | wc -l)." #+end_src *** Disable TurboBoost #+begin_src sh :results output :tangle scripts/disable_turboboost.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) if [ `lsmod | grep msr | wc -l` -ne 1 ]; then echo "The =msr= module is not loaded. It should be." exit 1; fi # Get the list of online cores ONLINECPUS=$(for CPU in $(find /sys/devices/system/cpu/ | grep -v cpu0 | grep cpu[0-9]*$); do [[ $(cat $CPU/online) -eq 1 ]] && echo $CPU; done | grep cpu[0-9]*$ | sed 's/.*cpu//') # Enable for PU in ${ONLINECPUS}; do sudo zsh -c "/usr/sbin/wrmsr -p${PU} 0x1a0 0x850089" done # Disable & Check for PU in ${ONLINECPUS}; do echo "Disabling turbo boost mode for PU $PU." sudo zsh -c "/usr/sbin/wrmsr -p${PU} 0x1a0 0x4000850089" TURBOBOOST=$(sudo zsh -c "/usr/sbin/rdmsr -p${PU} 0x1a0 -f 38:38") if [[ "0" = $TURBOBOOST ]]; then echo "Failed to disable turbo boost for PU number $cpu. Aborting." exit 1 fi done #+end_src *** CPU Selection #+begin_src sh :results output :tangle scripts/cpu_selection.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) function usage() { echo "Input: number of CPUs to be used" echo "Output: core identifiers (NUMA-aware)" echo "$0 "; } NCPUS=${1:-} if [ -z "$NCPUS" ]; then echo "Error: is empty" usage; exit; fi # Check total number of CPUs NUMCPU=$($DIR/hyperthreading.sh | head -n1 | sed 's/^This system has \([0-9]*\) CPUs.*$/\1/') if [ $NCPUS -gt $NUMCPU ]; then echo "You request $NCPUS CPUs, but this system has only $NUMCPU. Sorry." exit fi RES=$(lscpu | grep "NUMA\ node[0-9]* " | sed "s/^.*:[[:space:]]*//" | head -n${NCPUS} | tr '\n' ' ' | sed "s/ $//") FINAL=$(for i in $RES; do if [ $(echo $i | grep "-" | wc -l) -eq 0 ]; then #just replace commas by spaces echo $i | sed "s/,/ /g" else #use seq to replace core ranges by actual core ids S=$(echo $i | sed "s/-.*$//"); E=$(echo $i | sed "s/^.*-//"); seq $S $E fi done | tr '\n' ' ') echo "Use these: $FINAL" #+end_src *** Detect DVFS driver (acpi-cpufreq) #+begin_src sh :results output :tangle scripts/detect_acpidriver.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) function usage() { echo "Input: number of CPUs to be used" echo "Output: core identifiers (NUMA-aware)" echo "$0 "; } PRESENT=$(cpufreq-info | grep driver | uniq | grep acpi-cpufreq | wc -l) if [ $PRESENT -ne 1 ]; then exit 1; fi exit 0 #+end_src *** Set frequency to maximum #+begin_src sh :results output :tangle scripts/set_maximum_frequency.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) MAXFREQ=$(cpufreq-info | grep limits | sed -e "s/.*- //" -e "s/ //g" | uniq) # Get all online cores ONLINECPUS=$(for CPU in $(find /sys/devices/system/cpu/ | grep -v cpu0 | grep cpu[0-9]*$); do [[ $(cat $CPU/online) -eq 1 ]] && echo $CPU; done | grep cpu[0-9]*$ | sed 's/.*cpu//') # Core 0 is always online ONLINECPUS="0 ${ONLINECPUS}" for PU in ${ONLINECPUS}; do echo "Setting the frequency of PU ${PU} to ${MAXFREQ}" sudo cpufreq-set -c ${PU} -f ${MAXFREQ} echo "After setting to max, the frequency is now $(cat /sys/devices/system/cpu/cpu${PU}/cpufreq/scaling_cur_freq)" done #+end_src ** Configure multi-node experimental settings A script to control the experiment. - [X] Input is a file with node names For each machine: - [X] Disable turboboost - [X] Disable hyperthreading - [X] The =acpi_cpufreq= - [X] Check for its use - [X] Fix processor frequency to maximum - [X] Call the getinfo script after everything #+begin_src sh :results output :tangle scripts/control_experiment.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) function usage() { echo "$0 [dest]"; echo " where is a file with a list of hostnames, one per line"; echo " where is a directory where captured files should be moved to"; } ############################## # Parameter handling section # ############################## MACHINEFILE=${1:-} if [ -z "$MACHINEFILE" ]; then echo "Error: is empty" usage; exit; fi DEST=${2:-} if [ -z "$DEST" ]; then DEST="." fi ############################## # For every host # ############################## while IFS='' read -u10 -r HOST; do # Ignore comments and empty lines [[ "$(echo $HOST | sed 's/ //g')" =~ ^$ ]] && continue [[ "$HOST" =~ ^#.*$ ]] && continue # Clear anything after the colon HOST=$(echo $HOST | sed "s/:.*$//g") echo "** $HOST" echo "*** passwordless ssh" # Check is passwordless ssh is possible X=$(eval ssh -o 'PreferredAuthentications=publickey' ${HOST} "echo -n"; echo $?) if [ $X -ne 0 ]; then echo "passwordless ssh is not possible to ${HOST}." exit ; else echo "${HOST} is acessible through passwordless ssh." fi echo "*** Check presence of acpi-cpufreq driver" # Check for acpi-cpufreq scp $DIR/detect_acpidriver.sh ${HOST}:/tmp/ PRESENT=$(eval ssh ${HOST} "/tmp/detect_acpidriver.sh ; echo $?") if [ $PRESENT -ne 0 ]; then echo "The driver acpi-cpufreq is not available in ${HOST}" exit else echo "I believe the ${HOST} is using the acpi-cpufreq DVFS driver." fi echo "*** Disable hyperthreading" # Disable hyperthreading scp $DIR/hyperthreading.sh ${HOST}:/tmp/ scp $DIR/disable_hyperthreading.sh ${HOST}:/tmp/ ssh ${HOST} /tmp/disable_hyperthreading.sh echo "*** Disable turboboost" # Disable turboboost scp $DIR/disable_turboboost.sh ${HOST}:/tmp/ ssh ${HOST} /tmp/disable_turboboost.sh echo "*** Set the frequency to maximum" # Manually set the frequency to maximum scp $DIR/set_maximum_frequency.sh ${HOST}:/tmp/ ssh ${HOST} /tmp/set_maximum_frequency.sh echo "*** Call the get_info.sh" # After everything, call the get_info.sh script scp $DIR/get_info.sh ${HOST}:/tmp/ GETINFO=$(echo ${HOST} | sed "s/\./_/g" | sed "s/$/_get_info.org/") ssh ${HOST} "rm -f /tmp/${GETINFO}" ssh ${HOST} "/tmp/get_info.sh -t 'Get information from the ${HOST}' /tmp/${GETINFO}" scp ${HOST}:/tmp/$GETINFO . echo $GETINFO if [ "$DEST" != "." ]; then mv -f $GETINFO $DEST fi done 10< ${MACHINEFILE} #+end_src ** Run an Alya experiment Input: - [X] The machine file with the list of reserved machines - [X] The number of MPI processes to launch - [X] Path to Alya binary - [X] Path to the DAT file Everything should be ran in =/tmp/= directory, outside of NFS. The script will pushd to the directory of the DAT file (that should be outside of NFS) and ran from there. #+begin_src sh :results output :tangle scripts/run_alya_experiment.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) source $DIR/create_experiment_dir.sh function usage() { echo "$0 "; echo " where is a file with a list of hostnames, one per line"; echo " where is the number of MPI process"; echo " where is the path to the Alya binary"; echo " where is the path to the DAT file"; echo " where is a comment without white spaces"; echo " where (for infiniband) is true or false"; } function now() { date +%s; } ############################## # Parameter handling section # ############################## MACHINEFILE=${1:-} if [ -z "$MACHINEFILE" ]; then echo "Error: is empty" usage; exit; fi MACHINEFILE=$(readlink -f $MACHINEFILE) if [ ! -e "$MACHINEFILE" ]; then echo "Error: '$MACHINEFILE' does not exists." usage; exit fi NP=${2:-} if [ -z "$NP" ]; then echo "Error: is empty" usage; exit; fi ALYA=${3:-} if [ -z "$ALYA" ]; then echo "Error: is empty" usage; exit; fi ALYA=$(readlink -f $ALYA) DAT=${4:-} if [ -z "$DAT" ]; then echo "Error: is empty" usage; exit; fi DAT=$(readlink -f $DAT) UNIQUE=${5:-} if [ -z "$UNIQUE" ]; then echo "Error: is empty" usage; exit; else # Make sure unique is without spaces UNIQUE=$(echo $UNIQUE | sed "s/ /_/g") fi IB=${6:-} if [ -z "$IB" ]; then echo "Error: is empty"; exit; fi ############################## # Create the directory # ############################## # The org file of this script OUTPUTORG=$(org_file $UNIQUE) OUTPUTORGERR=$(org_err_file $UNIQUE) # The getinfo of where this script has been executed OUTPUTORGINFO=$(org_info_file $UNIQUE) # The place where results will be stored OUTPUTORGDIR=$(dir_file $UNIQUE) # Create the experimental directory create_experiment_dir $UNIQUE # Create the two org files touch $OUTPUTORG $OUTPUTORGINFO # From now on, all standard output goes to $OUTPUTORG exec > $OUTPUTORG exec 2> $OUTPUTORGERR DRYRUN=false ############################## # Prepare the experiment # ############################## echo "* Prepare the environment" # Call control the experiment if [ "$DRYRUN" = "false" ]; then $DIR/control_experiment.sh ${MACHINEFILE} ${OUTPUTORGDIR} else echo "This is a dry run, not calling the control experiment" fi # Save the machinefile forever cp $MACHINEFILE ${OUTPUTORGDIR} echo "* Alya" echo "** Binary" echo "The alya binary: $ALYA" ls -lh $ALYA # ldd is rather important ldd $ALYA # Heinrich told me he saves the binary, let's do the same cp $ALYA $OUTPUTORGDIR echo "** Input" echo "*** DAT" echo "The contents of the dat file:" ls -lh $DAT org_example_file $DAT DATDIR=$(dirname $DAT) DATBASE=$(basename $DAT .dat) echo "The DAT dir is ${DATDIR}, the DAT base is ${DATBASE}" echo "*** DOM.DAT" DOMDAT=${DATDIR}/${DATBASE}.dom.dat echo "The contents of the $DOMDAT" ls -lh $DOMDAT org_example_file $DOMDAT echo "*** KER.DAT" KERDAT=${DATDIR}/${DATBASE}.ker.dat echo "The contents of the $KERDAT" ls -lh $KERDAT org_example_file $KERDAT echo "*** NSI.DAT" NSIDAT=${DATDIR}/${DATBASE}.ker.dat echo "The contents of the $NSIDAT" ls -lh $NSIDAT org_example_file $NSIDAT if [ "$DRYRUN" = "false" ]; then $DIR/get_info.sh -t "Get information from launcher host $(hostname)" $OUTPUTORGINFO else echo "This is a dry run, not calling the get info script" fi ############################## # Run the experiment # ############################## echo "* ScoreP directory" SCOREPDIR="$HOME/scorep_${UNIQUE}" echo "$SCOREPDIR (finally moved to this experiment directory)" rm -rf $SCOREPDIR echo "* The Experiment" if [ "$IB" = "true" ]; then OPENIB="" else OPENIB="OMPI_MCA_btl=\"tcp,self,sm\"" fi # btl_base_verbose is set to 40 "info" COMMAND="${OPENIB} mpirun --mca btl_base_verbose 40 -x SCOREP_MPI_ENABLE_GROUPS=no -x SCOREP_VERBOSE=true -x SCOREP_TOTAL_MEMORY=4G -x SCOREP_EXPERIMENT_DIRECTORY=${SCOREPDIR} -x SCOREP_OVERWRITE_EXPERIMENT_DIRECTORY=true -x SCOREP_ENABLE_TRACING=true --bind-to core --report-bindings -np $NP -machinefile $MACHINEFILE $ALYA ${DATDIR}/${DATBASE}" echo "The command to be executed: '$COMMAND'" echo "The epoch time now, before the execution, is: $(now)" echo "#+BEGIN_EXAMPLE" if [ "$DRYRUN" = "false" ]; then eval $COMMAND fi echo "#+END_EXAMPLE" echo "The epoch time now, after the execution, is: $(now)" ############################## # Copy LOGs # ############################## echo "* The Logs" ls -lh ${DATDIR}/*.log cp ${DATDIR}/*.log $OUTPUTORGDIR echo "* Copy other small produced data" ls -lh ${DATDIR}/*.cvg ${DATDIR}/*.msh ${DATDIR}/*.res ${DATDIR}/*.sol ${DATDIR}/*.set cp ${DATDIR}/*.cvg ${DATDIR}/*.msh ${DATDIR}/*.res ${DATDIR}/*.sol ${DATDIR}/*.set $OUTPUTORGDIR echo "* Copy results of our manual instrumentation of alya code" ls -lh results_NPOIN_NELEM_NELEW_NBOUN.log mv -f results_NPOIN_NELEM_NELEW_NBOUN.log $OUTPUTORGDIR ############################## # Save in the directory # ############################## # Move scorep directory to mv -f $SCOREPDIR $OUTPUTORGDIR # Move org file to directory mv -f $OUTPUTORG $OUTPUTORGDIR # Move org err file to directory mv -f $OUTPUTORGERR $OUTPUTORGDIR # Move get_info org to directory mv -f $OUTPUTORGINFO $OUTPUTORGDIR # Copy myself to directory cp -f $0 $OUTPUTORGDIR # Copy all scripts to directory cp -f $DIR/*.sh $OUTPUTORGDIR # Exit graciously exit 0 #+end_src #+RESULTS: ** Convert traces.otf2 to CSV (version with =pj_dump=) #+begin_src sh :results output :tangle scripts/otf22csv.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) otf22csv() { pushd $(dirname $otf2) otf22paje traces.otf2 | pj_dump | grep ^State | cut -d, -f2,4,5,8 | sed -e "s/ //g" -e "s/MPIRank//" | gzip > traces.csv.gz popd } function usage() { echo "$0 " echo " where is a directory that contains .otf2 files"; } ############################## # Parameter handling section # ############################## OTF2DIR=${1:-} if [ -z "$OTF2DIR" ]; then echo "Error: is empty" usage; exit; fi OTF2DIR=$(readlink -f $OTF2DIR) if [ ! -d "$OTF2DIR" ]; then echo "Error: $OTF2DIR is not a directory" usage exit fi ############################## # Check for necessary tools # ############################## if [ -z "$(which pj_dump)" ]; then echo "Error: pj_dump is not in the PATH" exit fi if [ -z "$(which otf22paje)" ]; then echo "Error: otf22paje is not in the PATH" exit fi # Files already converted (whose CSV size is not zero) EXISTINGFILE=$(tempfile) OTF2FILE=$(tempfile) find $OTF2DIR -not -empty | grep csv.gz$ | sed -e "s/.csv.gz$//" | sort > $EXISTINGFILE find $OTF2DIR | grep \.otf2$ | sed -e "s/.otf2$//" | sort > $OTF2FILE for otf2 in $(comm -3 $OTF2FILE $EXISTINGFILE | sed "s/$/.otf2/"); do echo $otf2 otf22csv $otf2 done #+end_src ** Convert traces.otf2 to CSV (version *without* =pj_dump=, only *otf22csv*) #+begin_src sh :results output :tangle scripts/otf22csv_faster.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$(dirname $0) otf22csv_faster() { pushd $(dirname $otf2) otf22csv traces.otf2 | gzip > traces.csv.gz popd } function usage() { echo "$0 " echo " where is a directory that contains .otf2 files"; } ############################## # Parameter handling section # ############################## OTF2DIR=${1:-} if [ -z "$OTF2DIR" ]; then echo "Error: is empty" usage; exit; fi OTF2DIR=$(readlink -f $OTF2DIR) if [ ! -d "$OTF2DIR" ]; then echo "Error: $OTF2DIR is not a directory" usage exit fi ############################## # Check for necessary tools # ############################## if [ -z "$(which otf22csv)" ]; then echo "Error: otf22csv is not in the PATH" exit fi # Files already converted (whose CSV size is not zero) EXISTINGFILE=$(tempfile) OTF2FILE=$(tempfile) find $OTF2DIR -not -empty | grep csv.gz$ | sed -e "s/.csv.gz$//" | sort > $EXISTINGFILE find $OTF2DIR | grep \.otf2$ | sed -e "s/.otf2$//" | sort > $OTF2FILE for otf2 in $(comm -3 $OTF2FILE $EXISTINGFILE | sed "s/$/.otf2/"); do echo $otf2 otf22csv_faster $otf2 done #+end_src ** Post-processing in R #+begin_src R :results output :session :exports both :tangle scripts/trace2summary.R :tangle-mode (identity #o755) #!/usr/bin/Rscript library(readr); library(dplyr); alya_scorep_trace_read <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); read_csv(filename, col_names=c("Rank", "Start", "End", "Value"), progress=TRUE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, grepl("endste", .$Value) ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = as.integer(cumsum(Iteration))) %>% ungroup() %>% # Define metadata mutate(EID = meta[2], Platform = meta[3], Nodes = meta[4], NP = meta[5], Partitioning = meta[6], Infiniband = as.logical(meta[7])); } alya_scorep_trace_iterations <- function(filename) { alya_scorep_trace_read(filename) %>% group_by(Rank, Iteration, Platform, Nodes, NP, Partitioning, EID, Infiniband) %>% filter(grepl("MPI_", Value)) %>% summarize(N=n(), S=min(Start), E=max(End), Comm=sum(End-Start), Comp=(E-S)-Comm); } args = commandArgs(trailingOnly=TRUE) if (length(args) < 1) { stop("Usage: trace2summary.R ", call.=FALSE) } # # parse arguments # outputFilename = args[[1]]; inputs = unlist(args[2:length(args)]); df <- do.call("rbind", lapply(inputs, function(x) { alya_scorep_trace_iterations(x) })); write.csv(df, outputFilename); #+end_src * Guilherme entries * Meeting ** 2018-04-18 Arnaud and Lucas At http://rendez-vous.renater.fr/alyalbsfc Rick's goal: 10K cores in MN4 Eeach iteration has: Assemblage and Solver Rick has made a perfect LB in the Assembly phase only. The idea now is to make him repeat our observation methodology. ** 2018-04-18 Paper Outline (Discussion @ Barcelona w. Arnaud and Ricard) *** General information - Target: Physics journal/conf, multi-physics simulation developers. Note that I think this kind of work would totally make sense in CCPE or JPDC. - Thoughts about our work: - Nothing specific to Alya - Load balance in-depth study and mitigation is interesting as everyone relies on METIS or SFC. - Profiling technique not new for HPC performance experts but interesting for physics simulation developers. - 1D load balancing strategy not new from an algorithmic perspective - Refinement process expected to work but could have failed as well... - Doing a good load estimation and a good load balance in practice on a real code is not that easy common. - Goal: explain how this can be done as it could apply to many other codes *** Outline **** Intro **** Context :Guillaume:Ricard: - Alya structure - 3 typical test cases) - respiratory (up to ~1 K cpus - ~25M elem. mesh) - combustor (up to ~3K cpus - ~50M elem. mesh) - plane (up to ~6K cpus - 160M elem. mesh) - Scalability not satifying regardless of the partitioner in use (METIS, SFC). **** In-depth Load Analysis of one test case (mostly :Lucas:Arnaud: - Tracing is intrusive and cumbersome but an informed profiling with e.g., Score-P is cheap and requires almost no modification of the application. - Illustrate the load imbalance and regularity of the application across iterations. Illustrate the fact that the load imbalance is different for assembly and for the sparse solver. - Explain the reason behind the bad load balance (e.g., with SFC but this does not depend on the partitioner). There is no magic formula relating geometric characteristics with the actual execution time. **** Proposal: iterative improvement of the load distribution on an SFC :Lucas:Arnaud:Ricard: parition based on a few iterations of the application. - Illustration of the quick improvements of the load balance **** Evaluation of the technique on the three test cases at different scales :Lucas:Arnaud:Ricard: - Report both: - the parallel execution time - an evaluation of the load imbalance on the assembly and on the solver to know how much gain can still be expected - the amount of communications **** Conclusion - The main issue is the load estimation which seems very hard to do without running the code for real. - Our iterative load refinement allows significant gains but requires a few "dry runs". Ideally, it would be possible to rebalance the load at runtime, which is something on which we are working. *** Issues to solve (Ordered by priority) - Built-in standard profiling :: Ricard will run the experiments as he has more experience in this but we need systematic and precise load profiles for both writing the paper and deciding how to efficiently balance (e.g., balancing the assembly vs. balancing the sparse solver vs. both at the same time). To this end, Lucas should install score-p on MN4 and explain Ricard how to obtain the traces/profiles. Arnaud and Lucas will then do the analysis and graph generation. - Granularity problem :: Too coarse granularity is the best explanation (simpler than unstable memory ordering) for the lack of good performance at scale and the "jumps" we observed with Lucas. It seems there was some memory issue preventing Ricard to run with a finer granularity (bin size) - Pre-processing :: Solve pre-processing problems to run SFC on large cases. *** Possible future work - Could DLB be applied with our static iterative improvemet (e.g., by grouping heavily loaded nodes with lightly loaded ones if we cannot solve our granularity problem). Guillaume is not convinced as there is some overhead. - Would it make sense to have different bin sizes, i.e., an SFC more or less fine grained ? If so would it be possible to adjust the bin size when reading the data structure ? What about a simple quad-tree that would automatically split when bins are too large ? First, we would record where elements lie, and estimate their density locally (Note that this could be a pre-processing step independant of the execution of Alya...) then, we would merge the density from the different ranks. The density could then be used to determine at which granularity the SFC should be managed. * Temporal entries ** Old entries (from my journal) *** 2017-02-06 Alya Meeting :MEET: Two alternatives with just one physics model: - METIS partitioning - Space filling curves Tasks: 1. Check if there is really a difference 2. Compare the two .par files - With GID or ours (ggplot2) Sources of problems: - Load balancing problem - Too many communication - Badly distributed Entered on [2017-02-06 Mon 17:46] *** 2017-02-07 Installing and Testing GID :BOOT: http://www.gidhome.com/download/official-versions/linux64/ Download the =tar.gz= version: #+begin_src shell :results output cd /tmp/ wget -q ftp://www.gidhome.com/pub/GiD_Official_Versions/Linux/amd64/gid13.0.1-linux-x64.tar.gz #+end_src #+RESULTS: Extract it #+begin_src shell :results output cd /tmp/ tar xfzv gid13.0.1-linux-x64.tar.gz | head #+end_src #+RESULTS: #+begin_example gid13.0.1-x64/ gid13.0.1-x64/gid.exe gid13.0.1-x64/desktop_icons.tcl gid13.0.1-x64/scripts/ gid13.0.1-x64/scripts/StatusLabel.tcl gid13.0.1-x64/scripts/ContourColors.tcl gid13.0.1-x64/scripts/units.tcl gid13.0.1-x64/scripts/gdi/ gid13.0.1-x64/scripts/gdi/pkgIndex.tcl gid13.0.1-x64/scripts/DataWindows.tcl #+end_example Check if there is a binary: #+begin_src shell :results output cd /tmp/gid13.0.1-x64/ file gid file gid.exe #+end_src #+RESULTS: : gid: POSIX shell script, ASCII text executable : gid.exe: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.15, BuildID[sha1]=b5e042b34364226055f5446ec033407d51bac59c, stripped Let's go for the script, using the terminal. Tool works in my machine, I can read the examples, =render= to make them beautiful. I tried the motor example. Entered on [2017-02-07 Tue 09:57] *** 2017-02-07 Discussions avec Guillaume Houzeaux (Arnaud's Notes) :MEET: ***** Original notes - Ouverture de compte sur marenostrum pour Lucas. Guillaume s'en occupe et nous dit dès que c'est fait. En début de semaine prochaine normalement. - Question générale: Pourquoi les space filling curves sont tellement plus efficaces sur leur code que METHIS alors que tous les articles disent le contraire ? - Use case: système respiratoire. - visualisation (avec GiD cimne x64, téléchargeable en version démo sans problème) des mesh/sous domaine MPI avec (fichier .par) en 3D pour visualiser qui est qui et où ils sont - Là, le résultat de Methis montrait qu'un des process était responsable de zones très éloignées (au début du tube respiratoire et dans les bronches). Est-ce que ça explique le problème ? Pas clair... Il faudra qu'on regarde ça nous même. - Typical size: 256 cores, 5 secondes pour quelques itérations, ce qui devrait suffire si le partitionnement est déjà fait. - Faire le partitionnement une bonne fois pour toute et faire un restart d'Alya à partir de là. - Par rapport à http://bsccase02.bsc.es/alya/index.html#Solution_procedure, la description de Guillaume correspond à ce qui se passe dans "solve module 1". Il y a juste un module, pas de couplage. (M= moment, S=Schurr). - A priori, pas de lien entre les informations applicatives et ce qui apparaît dans une trace paraver - Outputs: - un .log - -partition.par.post.msh et -partition.par.post.res - Compilation METHIS #+BEGIN_SRC cd Alya cd thirdparties/metis-4.0; make ... cd Extecutables/unix; cat config.h ; # Guillaume will provide a typical one ./configure nastin parall ; make #+END_SRC - Run: - Guillaume will explain us how to do the partitionning and how to reuse them. - See the doc: http://bsccase02.bsc.es/alya/tutorial/partitions.html. It is actually up-to-date - Warning: master slave organization. Hence partition on 255 and run on 256 procs. - Dans gensap-nastin.dat, on peut définir le NUMBER_OF_STEPS - Input: ***** Follow-up I've checked the site: http://bsccase02.bsc.es/alya/index.html#Solution_procedure #+BEGIN_EXAMPLE - Read files, define mesh dependent arrays - Initial solution or read restart - Output and postprocess +--- do time steps ! | - Compute time step | - Begin a time step: update bc, etc. | | do blocks ---------------------------------+ | | | do coupling --------+ | Modules are grouped into blocks | - Solve module1 | | ! - Solve module2 | | ! ... | | Useful if some equations are coupled | - Check coupling cvg | | and others decoupled. Example | end do coupling --------+ | Block 1: Nastin - Turbul | | Block 2: Chemic: species are transported | Goto new block | | end do blocks ---------------------------------+ | | - End time step | - Output and postprocess | +--- end do time steps - Output and postprocess - End the run #+END_EXAMPLE Then, I've checked the step-by-step tutorial to partition: http://bsccase02.bsc.es/alya/tutorial/partitions.html The key line in my simple view now is this one: #+BEGIN_EXAMPLE TASK: ONLY_PREPROCESS, BINARY, SUBDOMAIN=num_domains #+END_EXAMPLE present in the =case.dat= configuration file for ALYA. Run the MPI program without =mpirun=, using a single process. Then, the resulting pre-processing stage can be registered in files that are only read, and the actual simulation can take place. *** 2017-02-24 Marenostrum account creation :MN3: ***** E-mails from Guillaume Houzeaux :ATTACH: :PROPERTIES: :Attachments: Screen%20Shot%202017-02-01%20at%2014.49.05.png Screen%20Shot%202017-02-01%20at%2014.49.31.png :ID: 789c7373-448e-44a3-9c5a-a0afa244280d :END: Two screen shots joined here. ***** Paper about load balancing in Alya :ATTACH: :PROPERTIES: :Attachments: IJHPCA.pdf :ID: 5439e62d-768a-4a98-a77f-4890b829c6ff :END: Load balancing of an MPI parallel unstructured CFD code using DLB and OpenMP. Marta Garcia-Gasulla 1 , Guillaume Houzeaux 1 , Antoni Artigues 1 , Jesús Labarta 12 and Mariano Vázquez 1. - unstructured meshes - the assembly step - a loop over the elements of the mesh to compute element matrix and right-hand side - an then to assemble them into global matrix and right-hand side - load balancing is an issue - mesh partitioning is never perfect - hardware can also be a random source of imbalance - approach - dynamic load balance library (OpenMP) \to idle MPI tasks by others ***** E-mail of account creation :ATTACH: :PROPERTIES: :Attachments: user_responsibilities.pdf 0769_001.pdf :ID: b16bcfa4-f3db-40da-b443-8db078c817e9 :END: We would like to inform you that the accounts of your activity in the supercomputer Marenostrum III have been created. Here you have the basic information to access and use the supercomputer. When accessing to the machine for the first time, an error message regarding ssh keys may appear. This is normal and you have to erase (mn1,mn2,mn3) entries from: =$HOME/.ssh/known_hosts= We have 3 login nodes for access to MareNostrum III machine {mn1,mn2,mn3}.bsc.es,all connections must be done through ssh (Secure SHell). #+BEGIN_EXAMPLE ssh username@mn1.bsc.es ssh username@mn2.bsc.es #+END_EXAMPLE The usernames associated to your project leaded by Jose M. Cela are: =Lucas Schnorr:bsc21835= The password will be sent in a separate mail with no subject unless you already had one for dl01.bsc.es. In order to change your password, you have to login at: =dl01.bsc.es= with the same username and password than in the cluster. Then, you have to use the 'passwd' command. The new password will become effective 10 minutes after the change. For security reasons the password must be changed the first time every user accesses the machine. MareNostrum III provides 5 interactive nodes. In order to gain access to each of them you need a ssh connection #+BEGIN_EXAMPLE ssh login5 #+END_EXAMPLE The login nodes serve as front ends and are used typically for editing, compiling, preparation and submition of batch executions. It is not permitted the execution of cpu-bound programs on these nodes, if some execution needs more cputime than the permitted, this needs to be done through the batch queue system. The basic limits in queues by job are 48 "wall clock" hours and 2048 cpus. If you need to submit a job exceeding 1024 cpus, then you must contact the support team to study the scalability and schedule its execution. No connections are allowed from inside Marenostrum III to the outside world, so all the file transfers have to be executed from your local machines and not inside Marenostrum III. Example to copy files or directories from MN to external machine: #+BEGIN_EXAMPLE scp -r {username}@dt01.bsc.es:"SOURCE_MN_directory" "DEST_directory" #+END_EXAMPLE Example to copy files or directories from external machine to MN: #+BEGIN_EXAMPLE scp -r "SOURCE_directory" {username}@dt01.bsc.es:"DEST_MN_directory" #+END_EXAMPLE Each project needs to inform us regularly about their evolution to let us know the status of each project running on the machine. All accounts will be for single user access and they must not be shared among several users. If your project needs more resources or accounts, only the project manager must do this kind of request. You can find the user's guide of MareNostrum III at the following link: http://www.bsc.es/support/MareNostrum3-ug.pdf Remember to send us the User Responsibilities document. If we do not receive it 15 days after the account is accepted, the access will be disabled. All BSC's staff is eager to help you in the development and success of your project, please contact us for any question / suggestion: e-mail: support@bsc.es Barcelona Supercomputing Center C/ Jordi Girona, 31 08034 Barcelona Entered on [2017-02-24 Fri 10:46] *** 2017-02-24 Trying to compile Alya, preprocess, postprocess :ALYA: **** Preliminaries (with Alya compilation) ***** Local SSH configuration for BSC supercomputer Put this in your =.ssh/config= #+BEGIN_EXAMPLE Host bscdata Hostname dt01.bsc.es User bsc21835 Host bsc Hostname mn3.bsc.es User bsc21835 #+END_EXAMPLE ***** Source code identification Currently in my Dropbox. It corresponds to a tarball Guillaume Houseaux sent me a couple of months ago (the same used by M. Camelo in his PFE). #+begin_src shell :results output cd ~/Downloads/ md5sum AlyaR1Source.tar.gz #+end_src #+RESULTS: : e8d20de91b56c64030d04ed1a4b1c559 AlyaR1Source.tar.gz ***** Copy Alya Sources to MN3 #+begin_src shell :results output rsync -avuzP ~/Downloads/AlyaR1Source.tar.gz bscdata:. ssh bscdata md5sum AlyaR1Source.tar.gz #+end_src #+RESULTS: : sending incremental file list : AlyaR1Source.tar.gz : 32,768 0% 0.00kB/s 0:00:00 21,856,256 21% 20.81MB/s 0:00:03 56,590,336 55% 26.98MB/s 0:00:01 93,487,104 92% 29.72MB/s 0:00:00 101,349,570 100% 29.91MB/s 0:00:03 (xfr#1, to-chk=0/1) : : sent 91,884,154 bytes received 34 bytes 20,418,708.44 bytes/sec : total size is 101,349,570 speedup is 1.10 : e8d20de91b56c64030d04ed1a4b1c559 AlyaR1Source.tar.gz ***** Alya Compilation Execute the following commands there. #+begin_src shell :results output tar xfz AlyaR1Source.tar.gz cd ~/AlyaR1Source/Thirdparties/metis-4.0; make cd ~/AlyaR1Source/Executables/unix/; cp configure.in/config_ifort.in config.in ./configure nastin parall ; make #+end_src I've got two binaries (g for Debug): #+BEGIN_EXAMPLE -rwxr-xr-x 1 bsc21835 bsc21 8.8M Feb 24 14:09 Alya.x -rwxr-xr-x 1 bsc21835 bsc21 27M Feb 24 14:10 Alya.g #+END_EXAMPLE **** Preprocess the =4_tufan= ***** Copy the test case to there #+begin_src shell :results output md5sum ~/Downloads/4_tufan_run.tar.gz rsync -avuzP ~/Downloads/4_tufan_run.tar.gz bscdata:. ssh bscdata md5sum 4_tufan_run.tar.gz #+end_src #+RESULTS: : b4127cb22bed9309d1234827f41135c9 /home/schnorr/Downloads/4_tufan_run.tar.gz : sending incremental file list : 4_tufan_run.tar.gz : 32,768 0% 0.00kB/s 0:00:00 20,709,376 5% 19.72MB/s 0:00:19 52,789,248 12% 25.16MB/s 0:00:13 82,837,504 20% 26.33MB/s 0:00:12 115,376,128 28% 27.51MB/s 0:00:10 149,848,064 36% 30.80MB/s 0:00:08 183,861,248 45% 31.27MB/s 0:00:07 216,989,696 53% 31.99MB/s 0:00:05 249,724,928 61% 32.04MB/s 0:00:04 283,770,880 69% 31.93MB/s 0:00:03 317,849,600 77% 31.95MB/s 0:00:02 347,111,424 84% 31.02MB/s 0:00:01 377,749,504 92% 30.53MB/s 0:00:00 408,525,195 100% 30.11MB/s 0:00:12 (xfr#1, to-chk=0/1) : : sent 396,055,909 bytes received 34 bytes 27,314,202.97 bytes/sec : total size is 408,525,195 speedup is 1.03 : b4127cb22bed9309d1234827f41135c9 4_tufan_run.tar.gz ***** Unpack #+begin_src shell :results output tar xfz 4_tufan_run.tar.gz cd 4_tufan_run/c/ #+end_src ***** Preprocess using METIS 4.0 Everything is done locally, in my notebook, because I don't know yet how to submit jobs in MN and Alya.x fails to run on =login= nodes. See: http://bsccase02.bsc.es/alya/tutorial/partitions.html - Partion preprocess step I am using Metis 4, but the tutorial tells me to use version 5. Let's proceed anyway. Create the directories to hold the partitioned meshes (for 16 domains) #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ ../../Utils/user/alya-hierarchy 116 #+end_src #+RESULTS: : : --| alya-hierarchy |-- : --| : --| 2 directories have been created : --| : --| Bye. : --| : --| This is the DAT file for preprocessing. #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ cat lucas_preprocess_case.dat #+end_src #+RESULTS: #+begin_example $------------------------------------------------------------------- RUN_DATA ALYA: sq_cyl RUN_TYPE: noCONTI , PRELIMINARY, FREQUENCY=100 LATEX_INFO_FILE: Yes LIVE_INFORMATION: Screen END_RUN_DATA $------------------------------------------------------------------- PROBLEM_DATA TIME_COUPLING: GLOBAL, PRESCR TIME_INTERVAL= 0.0,100000.0 TIME_STEP_SIZE= 0.00025 NUMBER_OF_STEPS= 100000 MAXIMUM_NUMBER_GLOBAL= 1 NASTIN_MODULE: On END_NASTIN_MODULE PARALL_SERVICE: On PARTITION_TYPE: FACES FILE_HIERARCHY: ON FILE_OPEN_CLOSE: Yes VIRTUAL_FILE: On, MAXIMUM_MEMORY=0.5 TASK: ONLY_PREPROCESS, BINARY, SUBDOMAIN=16 END_PARALL_SERVICE END_PROBLEM_DATA $------------------------------------------------------------------- #+end_example Note the customized =SUBDOMAIN=16= setting. Local execution gives this as output: #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ mpirun -np 1 ../../Executables/unix/Alya.x lucas_preprocess_case #+end_src #+RESULTS: #+begin_example --| --| ALYA START ALYA FOR PROBLEM: sq_cyl --| --| ALYA READ PROBLEM DATA --| ALYA START CONSTRUCT DOMAIN --| ALYA READ MESH DATA --| ALYA READ MESH ARRAYS --| ALYA READ ELEMENT TYPES --| ALYA CHECK ELEMENT TYPES --| ALYA READ ELEMENT CONNECTIVITY --| ALYA CHECK ELEMENT CONNECTIVITY --| ALYA READ COORDINATES --| ALYA READ BOUNDARY CONNECTIVITY, TYPES AND ELEMENT CONNECTIVITY --| ALYA CHECK BOUNDARY TYPES --| ALYA CHECK BOUNDARY CONNECTIVITY --| ALYA CHECK CONNECTIVITY BOUNDARY/ELEMENT --| ALYA READ BOUNDARY CONDITIONS --| ALYA WARNING: KERNEL WILL EXTRAPOLATE FROM BOUNDARY CODES TO NODE CODES --| ALYA WARNING: KERNEL READS BOUNDARY CODES ON BOUNDARIES --| ALYA END READ MESH ARRAYS --| ALYA COMPUTE GRAPH --| ALYA KERNEL: COMPUTE GROUPS OF DEFLATED CG --| ALYA START MESH PARTITION (# OF SUBDOMAINS= 16) --| ALYA PARALL PREPROCESS --> ALLOCATE MEMORY... --| ALYA PARALL PREPROCESS --> SET WEIGHTS... --| ALYA MASTER COMPUTES ELEMENT GRAPH (# EDGES= 6214430, MAX # EDGES/ELEMENT= 6) --| ALYA MASTER PARTITIONS ELEMENT GRAPH USING FACE CONNECTIVITY --| ALYA MASTER COMPUTES COMMUNICATION STRATEGY --| ALYA MASTER ORDERS INTERIOR AND BOUNDARY NODES --| ALYA MASTER COMPUTES PERMUTATION ARRAYS --| ALYA MASTER WRITES MESH AND PARTITION DATA IN RESTART FILES WITH HIERARCHY --| ALYA PARALL PREPROCESS --> S/R 1 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 2 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 3 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 4 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 5 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 6 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 7 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 8 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 9 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 10 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 11 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 12 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 13 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 14 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 15 - TOT 16 --| ALYA PARALL PREPROCESS --> S/R 16 - TOT 16 --| ALYA PARALL PREPROCESS --> SEND/REC 16 - TOTAL 16 --| ALYA PARALL PREPROCESS --> COORD, LTYPE, LNODS, LTYPB, LNODB... --| ALYA PARALL PREPROCESS --> LBOEL... --| ALYA PARALL PREPROCESS --> KFL_FIELD, XFIEL, TIME_FIELD,... --| ALYA PARALL PREPROCESS --> SET DATA... --| ALYA PARALL PREPROCESS --> BC DATA... --| ALYA PARALL PREPROCESS --> COMMUNICATION ARRAYS... --| ALYA PARALL PREPROCESS --> SEND DATA TO SLAVES... DONE. --| ALYA END MESH PARTITION --| --| ALYA CALCULATIONS CORRECT --| #+end_example Here are the contents of the =PAR*= dirs: #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ ls -1hl PAR000* #+end_src #+RESULTS: #+begin_example PAR000000: total 4.0K -rw-r--r-- 1 schnorr schnorr 2.9K Feb 24 16:02 lucas_preprocess_case.par.rst0 PAR000100: total 102M -rw-r--r-- 1 schnorr schnorr 6.4M Feb 24 16:02 lucas_preprocess_case.par.rst1 -rw-r--r-- 1 schnorr schnorr 6.4M Feb 24 16:02 lucas_preprocess_case.par.rst10 -rw-r--r-- 1 schnorr schnorr 6.5M Feb 24 16:02 lucas_preprocess_case.par.rst11 -rw-r--r-- 1 schnorr schnorr 6.4M Feb 24 16:02 lucas_preprocess_case.par.rst12 -rw-r--r-- 1 schnorr schnorr 6.3M Feb 24 16:02 lucas_preprocess_case.par.rst13 -rw-r--r-- 1 schnorr schnorr 6.7M Feb 24 16:02 lucas_preprocess_case.par.rst14 -rw-r--r-- 1 schnorr schnorr 6.6M Feb 24 16:02 lucas_preprocess_case.par.rst15 -rw-r--r-- 1 schnorr schnorr 6.3M Feb 24 16:02 lucas_preprocess_case.par.rst16 -rw-r--r-- 1 schnorr schnorr 6.3M Feb 24 16:02 lucas_preprocess_case.par.rst2 -rw-r--r-- 1 schnorr schnorr 6.5M Feb 24 16:02 lucas_preprocess_case.par.rst3 -rw-r--r-- 1 schnorr schnorr 6.4M Feb 24 16:02 lucas_preprocess_case.par.rst4 -rw-r--r-- 1 schnorr schnorr 6.3M Feb 24 16:02 lucas_preprocess_case.par.rst5 -rw-r--r-- 1 schnorr schnorr 6.5M Feb 24 16:02 lucas_preprocess_case.par.rst6 -rw-r--r-- 1 schnorr schnorr 6.3M Feb 24 16:02 lucas_preprocess_case.par.rst7 -rw-r--r-- 1 schnorr schnorr 6.3M Feb 24 16:02 lucas_preprocess_case.par.rst8 -rw-r--r-- 1 schnorr schnorr 6.6M Feb 24 16:02 lucas_preprocess_case.par.rst9 #+end_example Okay, now let's try to use them avoiding the preprocess. **** Postprocess the =4_tufan= ***** Postprocess Still local (notebook), for the same reasons. See: http://bsccase02.bsc.es/alya/tutorial/partitions.html - Partion postprocess step #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ cat lucas_postprocess_case.dat #+end_src #+RESULTS: #+begin_example $------------------------------------------------------------------- RUN_DATA ALYA: sq_cyl RUN_TYPE: noCONTI , PRELIMINARY, FREQUENCY=100 LATEX_INFO_FILE: Yes LIVE_INFORMATION: Screen END_RUN_DATA $------------------------------------------------------------------- PROBLEM_DATA TIME_COUPLING: GLOBAL, PRESCR TIME_INTERVAL= 0.0,100000.0 TIME_STEP_SIZE= 0.00025 NUMBER_OF_STEPS=2 MAXIMUM_NUMBER_GLOBAL= 1 NASTIN_MODULE: On END_NASTIN_MODULE $ this is the section that does the postprocess PARALL_SERVICE: On PARTITION_TYPE: FACES FILE_HIERARCHY: ON FILE_OPEN_CLOSE: Yes VIRTUAL_FILE: On, MAXIMUM_MEMORY=0.5 TASK: READ_PREPROCESS, BINARY END_PARALL_SERVICE $ This part has been removed to only preprocess $ PARALL_SERVICE: On $ OUTPUT_FILE: OFF $ POSTPROCESS: MASTER $ PARTITION_TYPE: FACES $ $ COMMUNICATION: ASYNCRONOUS $ END_PARALL_SERVICE END_PROBLEM_DATA $------------------------------------------------------------------- #+end_example The machine file I have: #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ cat machinefile #+end_src #+RESULTS: : localhost slots=32 max_slots=32 Now, let's run again with 17 procs: #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ mpirun -machinefile machinefile -np 17 ../../Executables/unix/Alya.x lucas_postprocess_case #+end_src #+RESULTS: This has failed. The log tells me why: #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ cat lucas_postprocess_case.log #+end_src #+RESULTS: #+begin_example -------- |- A L Y A: sq_cyl -------- -------------------------------------------------------- |- AN ERROR HAS BEEN DETECTED: ERROR WHEN OPENING THE PARALL RESTART FILE: PAR000000/lucas_postprocess_case.par.rst0 #+end_example Looks like I can't change the DAT filename. Let's rename it. Next code block is non-reproducible. #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ #cp lucas_preprocess_case.dat lucas_preprocess_case.dat.bak #cp lucas_postprocess_case.dat lucas_preprocess_case.dat cat lucas_preprocess_case.dat #+end_src #+RESULTS: #+begin_example $------------------------------------------------------------------- RUN_DATA ALYA: sq_cyl RUN_TYPE: noCONTI , PRELIMINARY, FREQUENCY=100 LATEX_INFO_FILE: Yes LIVE_INFORMATION: Screen END_RUN_DATA $------------------------------------------------------------------- PROBLEM_DATA TIME_COUPLING: GLOBAL, PRESCR TIME_INTERVAL= 0.0,100000.0 TIME_STEP_SIZE= 0.00025 NUMBER_OF_STEPS=2 MAXIMUM_NUMBER_GLOBAL= 1 NASTIN_MODULE: On END_NASTIN_MODULE $ this is the section that does the postprocess PARALL_SERVICE: On OUTPUT_FILE: OFF PARTITION_TYPE: FACES FILE_HIERARCHY: ON FILE_OPEN_CLOSE: Yes VIRTUAL_FILE: On, MAXIMUM_MEMORY=0.5 TASK: READ_PREPROCESS, BINARY END_PARALL_SERVICE END_PROBLEM_DATA $------------------------------------------------------------------- #+end_example Okay. Let's run it: #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ mpirun -machinefile machinefile -np 17 ../../Executables/unix/Alya.x lucas_preprocess_case 2>&1 > x.log #+end_src #+RESULTS: #+begin_src shell :results output cd ~/svn/guilherme/alya-imag/4_tufan_run/7/ cat x.log #+end_src #+RESULTS: #+begin_example --| --| ALYA START ALYA FOR PROBLEM: sq_cyl --| --| ALYA CHECK MPI. 1 MASTER + 16 SUBDOMAINS --| ALYA MPI IS WORKING WELL --| ALYA READ PROBLEM DATA --| ALYA START CONSTRUCT DOMAIN --| ALYA MASTER/SLAVES READ MESH AND PARTITION DATA FROM RESTART FILE WITH HIERARCHY --| ALYA MASTER/SLAVES: ALL SLAVES HAVE READ THEIR RESTART FILES --| ALYA KERMOD: READ DATA --| ALYA KERMOD FILE DOES NOT EXITS: USE DEFAULT OPTIONS --| ALYA PARALL: COMPUTE INTER-COLOR COMMUNICATORS --| ALYA PARALL: COMPUTE BIN STRUCTURE FOR PARTITIONS --| ALYA RENUMBER ELEMENTS --| ALYA RECOMPUTE GRAPH FOR PERIODICITY --| ALYA CHECK ELEMENT ORDERING --| ALYA CHECK BOUNDARY ORDERING --| ALYA PARALL: GET HALO GEOMETRY --| ALYA PARALL: CREATE INTRA-ZONE COMMUNICATION ARRAY --| ALYA PARALL: CREATE INTRA-SUBDOMAIN COMMUNICATION ARRAY --| ALYA OUTPUT MESH --| ALYA COMPUTE LUMPED MASS MATRIX --| ALYA COMPUTE CLOSED MASS MATRIX --| ALYA COMPUTE EXTERIOR NORMALS --| ALYA EXTRAPOLATE BOUNDARY CODES TO NODE CODES --| ALYA END CONSTRUCT DOMAIN --| ALYA MODULE DATA --| ALYA NASTIN: READ DATA --| ALYA NASTIN: MASTER SENDS PHYSICAL AND NUMERICAL DATA TO SLAVES --| ALYA NASTIN: MASTER SENDS PHYSICAL AND NUMERICAL ARRAYS TO SLAVES --| ALYA WARNINGS HAVE BEEN FOUND IN MODULE NASTIN --| --| ALYA ABORTED IN |----> 1 ERROR HAS BEEN FOUND IN NASTIN DATA FILE --| #+end_example **** Next steps - Get an updated version of Alya - Fix the partition postprocess step - Test case - The mesh for the respiratory system - Adapt preprocess + postprocess to this case - Find out how to activate the second method for mesh partitioning (space filling curves) - Reconfirm the fact that SFC are faster than METIS ** 2017-03-27 Trying to compile SVN version of Alya locally *** Configure with gfortran #+begin_src shell :results output cd ~/misc/alya/alya-bsc/Executables/unix/ rm config.in ln -s ./configure.in/config_gfortran.in config.in ./configure nastin parall #+end_src #+RESULTS: #+begin_example --| Alya configure |-- --| Build Alya makefile Linux OS identified. Using Intel Fortran Compiler ifort. Creating makefile for the modules demanded, when the source folder exists: Name: adapti ( service # 59 ) --> compile = no iffold = yes iffake = no Name: alefor ( module # 7 ) --> compile = no iffold = yes iffake = no Name: apelme ( module # 9 ) --> compile = no iffold = no iffake = no Name: casimi ( module # 23 ) --> compile = no iffold = no iffake = no Name: cgns24 ( service # 56 ) --> compile = no iffold = yes iffake = yes Name: chemic ( module # 19 ) --> compile = no iffold = yes iffake = no Name: codire ( module # 3 ) --> compile = no iffold = yes iffake = no Name: commdo ( service # 65 ) --> compile = no iffold = no iffake = no Name: coupli ( service # 64 ) --> compile = no iffold = no iffake = no Name: dodeme ( service # 57 ) --> compile = no iffold = yes iffake = yes Name: exmedi ( module # 5 ) --> compile = no iffold = yes iffake = no Name: gidpos ( service # 53 ) --> compile = no iffold = yes iffake = yes Name: gotita ( module # 11 ) --> compile = no iffold = yes iffake = no Name: handfp ( service # 54 ) --> compile = no iffold = yes iffake = yes Name: hdfpos ( service # 62 ) --> compile = no iffold = yes iffake = no Name: helmoz ( module # 20 ) --> compile = no iffold = yes iffake = no Name: immbou ( module # 21 ) --> compile = no iffold = yes iffake = no Name: latbol ( module # 8 ) --> compile = no iffold = no iffake = no Name: levels ( module # 14 ) --> compile = no iffold = yes iffake = no Name: magnet ( module # 16 ) --> compile = no iffold = yes iffake = no Name: nasedg ( module # 18 ) --> compile = no iffold = yes iffake = no Name: nastal ( module # 6 ) --> compile = no iffold = yes iffake = no Name: nastin ( module # 1 ) --> compile = yes iffold = yes iffake = no --> OK to compile! Name: neutro ( module # 26 ) --> compile = no iffold = yes iffake = no Name: optsol ( service # 63 ) --> compile = no iffold = yes iffake = yes Name: parall ( service # 55 ) --> compile = yes iffold = yes iffake = yes --> OK to compile! Name: partis ( module # 17 ) --> compile = no iffold = yes iffake = no Name: porous ( module # 24 ) --> compile = no iffold = yes iffake = no Name: quanty ( module # 15 ) --> compile = no iffold = yes iffake = no Name: radiat ( module # 22 ) --> compile = no iffold = yes iffake = no Name: shapar ( service # 51 ) --> compile = no iffold = no iffake = no Name: solidz ( module # 10 ) --> compile = no iffold = yes iffake = no Name: solmum ( service # 52 ) --> compile = no iffold = yes iffake = yes Name: solpls ( service # 58 ) --> compile = no iffold = yes iffake = no Name: temper ( module # 2 ) --> compile = no iffold = yes iffake = no Name: turbul ( module # 4 ) --> compile = no iffold = yes iffake = no Name: wavequ ( module # 12 ) --> compile = no iffold = yes iffake = no Name: xxxxxx ( module # 25 ) --> compile = no iffold = no iffake = no Makefile will make the following binaries: Alya.x --> Production (non debugger) binary Alya.g --> Debugger binary --| Makefile created. Ready to make. --| The command you typed has been saved in file conf.log --| Bye. #+end_example *** Make #+begin_src shell :results output cd ~/misc/alya/alya-bsc/ThirdParties/metis-4.0/; make -j 4 2>&1 cd ~/misc/alya/alya-bsc/Executables/unix/; make -j 4 2>&1 #+end_src #+RESULTS: ** 2017-03-28 OAR Job Allocation #+begin_src shell :results output oarsub -p "cluster='taurus'" -l "nodes=4/walltime=03:00:00" -r "$(date '+%Y-%m-%d %H:%M:%S')" -t deploy oarsub -C 852259 #taurus-1.lyon.grid5000.fr #taurus-16.lyon.grid5000.fr #taurus-2.lyon.grid5000.fr #taurus-7.lyon.grid5000.fr kadeploy3 --env-file images/stretch_energy.env --file ${OAR_NODE_FILE} -k cd ~/alya-bsc/Thirdparties/metis-4.0/; make -j 4 cd ~/alya-bsc/Executables/unix/; ln -s ./configure.in/config_gfortran.in config.in; make -j 4 #+end_src ** 2017-03-29 Experiments to execute #+begin_src shell :results output for CASE in sfc metis; do for NP in 32 64 96 120; do NP=$((NP+1)) UNIQUE=taurus-np${NP}-${CASE} echo "./scripts/run_alya_experiment.sh machine-file ${NP} ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE" echo "mv $UNIQUE /home/lschnorr/" done done > script.sh cat script.sh #+end_src #+RESULTS: #+begin_example ./scripts/run_alya_experiment.sh machine-file 33 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np33-sfc mv taurus-np33-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 65 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np65-sfc mv taurus-np65-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 97 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np97-sfc mv taurus-np97-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 121 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np121-sfc mv taurus-np121-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 33 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np33-metis mv taurus-np33-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 65 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np65-metis mv taurus-np65-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 97 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np97-metis mv taurus-np97-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 121 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np121-metis mv taurus-np121-metis /home/lschnorr/ #+end_example ** 2017-03-29 Quick analysis of first results I got these: #+name: first-results #+begin_src shell :results output org table for case in $(find $HOME/segundo/taurus-np* | grep -e "sfc.org" -e "metis.org"); do echo -n "$case "; cat $case | grep epoch | sed "s/^.*: //" | tr '\n' '-' | sed -e "s/-$//" -e "s/-/ - /" -e "s/$/\n/" | bc -l; done | sed -e "s/^.*np//" -e "s/.org//" -e "s/-/ /g" -e "s/ */|/g" #+end_src #+RESULTS: first-results #+BEGIN_SRC org | 121 | metis | 0 | | 121 | sfc | 1 | | 33 | metis | 1160 | | 33 | sfc | 938 | | 65 | metis | 653 | | 65 | sfc | 612 | | 97 | metis | 503 | | 97 | sfc | 470 | #+END_SRC #+header: :var df=first-results #+begin_src R :results output graphics :file img/first-results.png :exports both :width 600 :height 400 :session library(dplyr); library(ggplot2); df %>% rename(NP = V1, Case = V2, Time = V3) %>% filter(NP != 121) %>% ggplot(aes(x=NP, y=Time, color=Case)) + theme_bw() + ylim(0,NA) + geom_point() + geom_line() + theme(legend.position = "top"); #+end_src #+RESULTS: [[file:img/first-results.png]] ** 2017-03-29 Planning the tracing with Scorep Score-P 3.0 allows the selective instrumentation of any function using a GCC plugin. Do to so, refer to the Section 3.1 of ScoreP user manual. The flag is =--instrument-filter=, much better described in Section 5.3 of the same document. This method is preferred since it reduces the intrusion. To make such method available in a scorep installation, we need to previously install the package =gcc-6-plugin-dev= or something similar. It is interesting to disable everything else which is not MPI and the necessary to trace filtered user functions. Guilherme, for example, had used this line to instrument Alya: =scorep --nopomp --nocompiler --nocuda --nopdt --nouser --noopencl= We would have to keep only: =scorep --nopomp --nocuda --noopencl --nopdt= This is the filter that Guilherme used dynamically. We want to use it statically. #+begin_src shell :results output :tangle alya-timestep-filters.scorep SCOREP_REGION_NAMES_BEGIN EXCLUDE * INCLUDE MPI COM INCLUDE endste_ timste_ SCOREP_REGION_NAMES_END #+end_src The way to use this filter in runtime is to use the appropriate environment variable this way: #+BEGIN_EXAMPLE SCOREP_FILTERING_FILE=name_of_the_file #+END_EXAMPLE _Summary_ of that has worked: Finally, I've managed to get it working with the following steps. 1. Configure ScoreP with the following line #+BEGIN_EXAMPLE ./configure --without-gui --prefix=/home/lschnorr/install/nova/scorep-3.0-alya/ --with-shmem=no --with-opari2=no #+END_EXAMPLE - Do not forget to install the gcc plugin package first 2. Configure Alya's =config.in= to use scorep this way #+BEGIN_EXAMPLE SCOREP=scorep --compiler --instrument-filter=/home/lschnorr/alya-bsc/Executables/unix/alya-timestep-filters.scorep #+END_EXAMPLE Where the contents of the filter are like this #+BEGIN_EXAMPLE #SCOREP_FILE_NAMES_BEGIN #EXCLUDE * #INCLUDE */Alya.f90 #SCOREP_FILE_NAMES_END SCOREP_REGION_NAMES_BEGIN EXCLUDE * INCLUDE endste INCLUDE timste SCOREP_REGION_NAMES_END #+END_EXAMPLE 3. Finally, run the program this way #+BEGIN_EXAMPLE mpirun -x SCOREP_FILTERING_FILE=/home/lschnorr/alya-bsc/Executables/unix/alya-timestep-filters.scorep -x SCOREP_EXPERIMENT_DIRECTORY=/home/lschnorr/xxx-scorep7 -x SCOREP_OVERWRITE_EXPERIMENT_DIRECTORY=true -x SCOREP_ENABLE_TRACING=true --bind-to core -np 12 -machinefile /tmp/Alya-Perf/machine-file /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap #+END_EXAMPLE - The traces need to be put in the HOME NFS. - Filtering is not obligatory here since we instrumented in compile time. ** 2017-04-05 Infiniband problems In all machines that belong to an Infiniband experiment, do: 1. In file =/etc/security/limits.conf=, add these lines: #+BEGIN_EXAMPLE * hard memlock unlimited * soft memlock unlimited #+END_EXAMPLE 2. In file =/etc/systemd/system.conf=, make the line looks like this: #+BEGIN_EXAMPLE DefaultLimitMEMLOCK=infinity #+END_EXAMPLE ** 2017-04-06 Nose check #+name: nose_table | Rank | X | Y | Z | |------+--------------+--------------+--------------| | 1 | 0.919827E-01 | 0.115893E+00 | -.639115E-01 | | 2 | 0.103425E+00 | 0.149886E+00 | -.123160E+00 | | 3 | 0.103773E+00 | 0.169636E+00 | -.146991E+00 | | 4 | 0.961346E-01 | 0.202625E+00 | -.216010E+00 | | 5 | 0.139644E+00 | 0.204323E+00 | -.223233E+00 | | 6 | 0.101879E+00 | 0.850899E-01 | 0.235732E-01 | | 7 | 0.933165E-01 | 0.458008E-01 | 0.261210E-01 | | 8 | 0.913297E-01 | 0.682040E-01 | 0.290498E-01 | | 9 | 0.952650E-01 | 0.111200E+00 | 0.149948E-01 | | 10 | 0.950596E-01 | 0.122169E+00 | -.234605E-01 | | 11 | -.403059E+00 | -.400000E+00 | -.489343E+00 | | 12 | 0.596941E+00 | -.400000E+00 | -.489343E+00 | | 13 | 0.596941E+00 | 0.240397E+00 | -.489343E+00 | | 14 | -.403059E+00 | 0.240397E+00 | -.489343E+00 | | 15 | -.403059E+00 | -.400000E+00 | 0.510656E+00 | | 16 | 0.596941E+00 | -.400000E+00 | 0.510656E+00 | | 17 | 0.596941E+00 | 0.240397E+00 | 0.510656E+00 | | 18 | -.403059E+00 | 0.240397E+00 | 0.510656E+00 | #+begin_src R :results output :session :exports both :var df=nose_table df; #+end_src #+RESULTS: #+begin_example Rank X Y Z 1 1 0.0919827 0.1158930 -0.0639115 2 2 0.1034250 0.1498860 -0.1231600 3 3 0.1037730 0.1696360 -0.1469910 4 4 0.0961346 0.2026250 -0.2160100 5 5 0.1396440 0.2043230 -0.2232330 6 6 0.1018790 0.0850899 0.0235732 7 7 0.0933165 0.0458008 0.0261210 8 8 0.0913297 0.0682040 0.0290498 9 9 0.0952650 0.1112000 0.0149948 10 10 0.0950596 0.1221690 -0.0234605 11 11 -0.4030590 -0.4000000 -0.4893430 12 12 0.5969410 -0.4000000 -0.4893430 13 13 0.5969410 0.2403970 -0.4893430 14 14 -0.4030590 0.2403970 -0.4893430 15 15 -0.4030590 -0.4000000 0.5106560 16 16 0.5969410 -0.4000000 0.5106560 17 17 0.5969410 0.2403970 0.5106560 18 18 -0.4030590 0.2403970 0.5106560 #+end_example #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session library(ggplot2); plot(df); #+end_src #+RESULTS: [[file:/tmp/babel-22618hTf/figure22618i_y.png]] ** 2017-04-06 Alya modifications to get more info, set balance to 1 :EXP10: #+BEGIN_EXAMPLE lschnorr@fnancy:~/alya-bsc$ git diff diff --git a/Executables/unix/configure.in/config_gfortran.in b/Executables/unix/configure.in/config_gfortran.in index 1c73a1a..90de238 100644 --- a/Executables/unix/configure.in/config_gfortran.in +++ b/Executables/unix/configure.in/config_gfortran.in @@ -4,16 +4,17 @@ #module load gcc/5.1.0 openmpi/1.8.5 # ################################################################### +SCOREP=scorep --compiler --instrument-filter=/home/lschnorr/alya-bsc/Executables/unix/alya-timestep-filters.scorep -F77 = mpif90 -F90 = mpif90 -FCOCC = mpicc -c +F77 = $(SCOREP) mpif90 +F90 = $(SCOREP) mpif90 +FCOCC = $(SCOREP) mpicc -c FCFLAGS = -c -J$O -I$O -ffree-line-length-none -fimplicit-none FPPFLAGS = -x f95-cpp-input EXTRALIB = -lc EXTRAINC = -fa2p = mpif90 -c -x f95-cpp-input -DMPI_OFF -J../../Utils/user/alya2pos -I../../Utils/user/alya2pos -fa2plk = mpif90 -lc +fa2p = $(SCOREP) mpif90 -c -x f95-cpp-input -DMPI_OFF -J../../Utils/user/alya2pos -I../../Utils/user/alya2pos +fa2plk = $(SCOREP) mpif90 -lc ################################################################### # PERFORMANCE FLAGS # diff --git a/Sources/kernel/parall/mod_par_partit_sfc.f90 b/Sources/kernel/parall/mod_par_partit_sfc.f90 index ee6fd1d..e55185d 100644 --- a/Sources/kernel/parall/mod_par_partit_sfc.f90 +++ b/Sources/kernel/parall/mod_par_partit_sfc.f90 @@ -383,7 +383,7 @@ contains iboxf = lboxf(ielem) iboxl = lboxl(ielem) - iweig = real(ngaus(abs(ltype_par(ielem))),rp) + iweig = 1_rp !real(ngaus(abs(ltype_par(ielem))),rp) if( PAR_MY_PARMETIS_RANK /= iboxc-1 )then if(bufwei(iboxl*2) == 0_rp) then diff --git a/Sources/services/parall/par_outinf.f90 b/Sources/services/parall/par_outinf.f90 index 6664b93..301522a 100644 --- a/Sources/services/parall/par_outinf.f90 +++ b/Sources/services/parall/par_outinf.f90 @@ -15,6 +15,7 @@ subroutine par_outinf() use mod_parall implicit none integer(ip) :: isubd,ksmin(2),ksmax(2),kaver,koutp,nb + integer, parameter :: out_unit_npoin=20 if( IMASTER ) then @@ -27,7 +27,11 @@ subroutine par_outinf() ksmin = huge(1_ip) ksmax = -huge(1_ip) kaver = 0 + ! LUCAS: manual instrumentation starts here, all code marked by LUCAS + open (unit=out_unit_npoin,file="results_NPOIN_NELEM_NELEW_NBOUN.log",action="write",status="replace") do isubd=1,npart_par + ! LUCAS: Here I need to print: npoin_par(isubd) + write (out_unit_npoin,*) "Rank ", isubd ," NPOIN ", npoin_par(isubd), " NELEM ", nelem_par(isubd), " NELEW ", nelew_par(isubd), " NBOUN ", nboun_par(isubd), " NNEIG ", lneig_par(isubd), " NBBOU ", npoin_par(isubd) if(npoin_par(isubd)% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df2 <- read_csv("data/11/326040-b8e2-41f3-8479-c378f70864d9/exp_11-v1_grimoire_8.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df <- rbind(df1, df2); df %>% head; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] # A tibble: 6 × 14 Rank Iteration Platform Nodes NP Partitioning EID Infiniband N 1 0 1 grimoire 8 128 sfc 10-v2 TRUE 5454 2 0 2 grimoire 8 128 sfc 10-v2 TRUE 4752 3 0 3 grimoire 8 128 sfc 10-v2 TRUE 4873 4 0 4 grimoire 8 128 sfc 10-v2 TRUE 4697 5 0 5 grimoire 8 128 sfc 10-v2 TRUE 4660 6 0 6 grimoire 8 128 sfc 10-v2 TRUE 4544 # ... with 5 more variables: Start , End , Comm , Comp , # Duration #+end_example **** Plot Comp #+begin_src R :results output graphics :file img/exp_10-v2_exp_11-v1_comp.png :exports both :width 500 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Comp", Iteration != 10, Rank != 0) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=Infiniband)) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(EID~Partitioning) #+end_src #+RESULTS: [[file:img/exp_10-v2_exp_11-v1_comp.png]] *** Partition data **** Process #+begin_src shell :results output EDIR="exp_11-v1_grimoire_8 exp_10-v2_grimoire_8" for file in $(find $EDIR | grep results | grep log$); do OUTPUT=$(dirname $file)/$(basename $file .log).csv cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f1,3,5,7,9,11,13 | uniq > $OUTPUT cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f2,4,6,8,10,12,14 >> $OUTPUT echo "=> $file <=" head $OUTPUT done #+end_src #+RESULTS: #+begin_example => exp_11-v1_grimoire_8/11-v1_grimoire_8_128_sfc_true.dir/results_NPOIN_NELEM_NELEW_NBOUN.log <= Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,32182,174168,174611,2973,16,32182 2,20919,53529,174758,5381,15,20919 3,19727,49972,175172,5020,11,19727 4,21918,69049,174054,4277,23,21918 5,21683,64345,174280,4439,14,21683 6,22205,64122,175002,4582,18,22205 7,21361,63432,174587,4436,11,21361 8,22162,75699,174459,3914,8,22162 9,21389,66339,174799,4332,8,21389 => exp_11-v1_grimoire_8/11-v1_grimoire_8_128_metis_true.dir/results_NPOIN_NELEM_NELEW_NBOUN.log <= Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,22405,85371,172276,3477,4,22405 2,22923,86049,179769,3742,5,22923 3,24316,99541,175416,3029,5,24316 4,22907,90905,170125,3170,4,22907 5,22636,86964,173194,3448,5,22636 6,22774,88416,171711,3333,6,22774 7,23564,91625,177865,3452,7,23564 8,22211,83394,173364,3612,5,22211 9,23427,89980,179685,3588,4,23427 => exp_10-v2_grimoire_8/10-v2_grimoire_8_128_sfc_true.dir/results_NPOIN_NELEM_NELEW_NBOUN.log <= Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,14976,72536,72796,2384,17,14976 2,14641,72392,72575,448,12,14641 3,22264,72647,167221,4446,16,22264 4,28129,72464,251684,7187,11,28129 5,23239,72488,178813,4341,25,23239 6,26093,72525,209615,5640,14,26093 7,22996,72391,183401,4422,10,22996 8,21190,72393,160643,3505,7,21190 9,25178,72612,201462,5195,10,25178 => exp_10-v2_grimoire_8/10-v2_grimoire_8_128_metis_true.dir/results_NPOIN_NELEM_NELEW_NBOUN.log <= Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,22405,85371,172276,3477,4,22405 2,22923,86049,179769,3742,5,22923 3,24316,99541,175416,3029,5,24316 4,22907,90905,170125,3170,4,22907 5,22636,86964,173194,3448,5,22636 6,22774,88416,171711,3333,6,22774 7,23564,91625,177865,3452,7,23564 8,22211,83394,173364,3612,5,22211 9,23427,89980,179685,3588,4,23427 #+end_example **** Read #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_npoin <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[2], "_")); meta <- gsub(".dir", "", meta); read_csv(filename) %>% gather(Variable, Value, -Rank) %>% mutate(EID = meta[1], Platform = meta[2], Nodes = meta[3], NP = meta[4], Partitioning = meta[5], Infiniband = as.logical(meta[6])); } files <- list.files("exp_10-v2_grimoire_8", pattern="results_NPOIN_NELEM_NELEW_NBOUN.csv", recursive=TRUE, full.names=TRUE); files <- c(files, list.files("exp_11-v1_grimoire_8", pattern="results_NPOIN_NELEM_NELEW_NBOUN.csv", recursive=TRUE, full.names=TRUE)); dfp <- do.call("rbind", lapply(files, function(x) { read_npoin(x) })) dfp %>% filter(Rank == 111, Variable == "NELEW") #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) # A tibble: 4 × 9 Rank Variable Value EID Platform Nodes NP Partitioning Infiniband 1 111 NELEW 172833 10-v2 grimoire 8 128 metis TRUE 2 111 NELEW 258786 10-v2 grimoire 8 128 sfc TRUE 3 111 NELEW 172833 11-v1 grimoire 8 128 metis TRUE 4 111 NELEW 173994 11-v1 grimoire 8 128 sfc TRUE #+end_example *** Merge Get only the first iteration #+begin_src R :results output :session :exports both dfz <- df %>% filter(Iteration == 1, Rank != 0) %>% select(-Platform, -Nodes, -NP, -Infiniband, -N, -Start, -End, -Comm, -Duration, -Iteration) %>% gather(Variable, Value, Comp, -EID, -Partitioning) %>% select(EID, Partitioning, Rank, Variable, Value); dfpz <- dfp %>% select(-Platform, -Nodes, -NP, -Infiniband) %>% select(EID, Partitioning, Rank, Variable, Value); dfm <- rbind(dfz, dfpz); dfm %>% head; #+end_src #+RESULTS: : # A tibble: 6 × 5 : EID Partitioning Rank Variable Value : : 1 10-v2 sfc 1 Comp 8.811678 : 2 10-v2 sfc 2 Comp 8.560127 : 3 10-v2 sfc 3 Comp 12.641175 : 4 10-v2 sfc 4 Comp 15.811419 : 5 10-v2 sfc 5 Comp 13.183812 : 6 10-v2 sfc 6 Comp 14.227162 *** Plot #+begin_src R :results output :session :exports both dfm %>% .$Variable %>% unique #+end_src #+RESULTS: : [1] "Comp" "NPOIN" "NELEM" "NELEW" "NBOUN" "NNEIG" "NBBOU" #+begin_src R :results output graphics :file img/exp_10-v2_exp_11-v1_comp_partition.png :exports both :width 800 :height 400 :session dfm %>% mutate(EID = case_when( grepl("10-v2", .$EID) ~ gsub("$", "(Equal Weight)", .$EID), grepl("11-v1", .$EID) ~ gsub("$", "(Original Weight)", .$EID), TRUE ~ "Undefined")) %>% filter(Partitioning != "metis") %>% filter(Variable %in% c("Comp", "NELEM", "NELEW")) %>% ggplot(aes(x=Rank, y=Value, color=EID)) + theme_bw(base_size=12) + geom_point() + ylim(0,NA) + ggtitle ("SFC only, NP=128, 8-nodes@grimoire, Infiniband") + theme(legend.position="top") + facet_grid(Variable~EID, scales="free_y") #+end_src #+RESULTS: [[file:img/exp_10-v2_exp_11-v1_comp_partition.png]] ** 2017-04-07 Check alternative Alya instrumentation :EXP12: *** First try - no imbrication #+begin_src shell :results output ssh nancy.g5k cat ./alya-bsc/Executables/unix/all.scorep #+end_src #+RESULTS: #+begin_example #SCOREP_FILE_NAMES_BEGIN #EXCLUDE * #INCLUDE */Alya.f90 #SCOREP_FILE_NAMES_END SCOREP_REGION_NAMES_BEGIN EXCLUDE * INCLUDE endste INCLUDE timste INCLUDE doiter INCLUDE concou INCLUDE conblk INCLUDE begzon INCLUDE endzon INCLUDE moduls INCLUDE nastin INCLUDE parall SCOREP_REGION_NAMES_END #+end_example #+begin_src shell :results output scp nancy.g5k:./scorep_12-v1_grimoire_6_96_metis_true/traces.csv.gz . #+end_src #+RESULTS: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("traces.csv.gz", col_names=c("Rank", "Start", "End", "Value"), progress=FALSE); df %>% group_by(Value) %>% summarize(N=n()) #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) # A tibble: 10 × 2 Value N 1 begzon 96 2 conblk 96 3 concou 96 4 doiter 96 5 endste 96 6 endzon 96 7 moduls 1248 8 nastin 1152 9 parall 431331 10 timste 96 #+end_example Let's concentrate in nastin: #+begin_src R :results output :session :exports both df %>% filter(Rank == 61) #+end_src #+RESULTS: #+begin_example # A tibble: 4,542 × 4 Rank Start End Value 1 61 0.010045 0.010054 parall 2 61 0.012898 0.043018 parall 3 61 0.043031 0.051135 parall 4 61 0.051233 0.051235 moduls 5 61 0.051251 249.036677 parall 6 61 249.037003 249.094670 parall 7 61 249.094764 249.094993 parall 8 61 249.095035 249.095187 parall 9 61 249.095202 249.095204 parall 10 61 249.553567 249.709909 parall # ... with 4,532 more rows #+end_example I need imbrication to understand that. *** With imbrication, plot #+begin_src shell :results output scp nancy.g5k:./scorep_12-v1_grimoire_6_96_metis_true/traces.csv.gz . #+end_src #+RESULTS: #+begin_src R :results output :session :exports both df <- read_csv("traces.csv.gz", col_names=c("Nature", "Rank", "Type", "Start", "End", "Duration", "Imbrication", "Value"), progress=FALSE); df; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Nature = col_character(), Rank = col_integer(), Type = col_character(), Start = col_double(), End = col_double(), Duration = col_double(), Imbrication = col_double(), Value = col_character() ) # A tibble: 857,455 × 8 Nature Rank Type Start End Duration Imbrication Value 1 State 95 STATE 0.000267 0.000279 0.000012 0 parall 2 State 95 STATE 0.000290 0.001318 0.001029 0 parall 3 State 95 STATE 0.001326 0.032243 0.030917 0 parall 4 State 95 STATE 0.032352 0.032354 0.000002 0 moduls 5 State 95 STATE 0.032367 252.062136 252.029768 0 parall 6 State 95 STATE 252.062477 252.118046 0.055569 0 parall 7 State 95 STATE 252.118115 252.118401 0.000286 0 parall 8 State 95 STATE 252.118442 252.118595 0.000153 0 parall 9 State 95 STATE 252.118610 252.118644 0.000034 0 parall 10 State 95 STATE 252.722345 252.732470 0.010126 0 parall # ... with 857,445 more rows #+end_example #+begin_src R :results output :session :exports both dfplot %>% .$Imbrication %>% unique %>% sort startTime <- df %>% filter(Value == "timste") %>% arrange(Start) %>% slice(1) %>% .$Start startTime endTime <- df %>% filter(Value == "endste") %>% arrange(End) %>% slice(n()) %>% .$End endTime dfplot %>% filter(Start > startTime, End < endTime) %>% filter(Imbrication != 10) %>% filter(Value == "moduls") #+end_src #+RESULTS: #+begin_example [1] 0 [1] 259.3502 [1] 315.6014 # A tibble: 15 × 8 Nature Rank Type Start End Duration Imbrication Value 1 State 4 STATE 259.6966 259.6972 0.000613 0 moduls 2 State 4 STATE 287.6941 287.7119 0.017868 0 moduls 3 State 4 STATE 288.0547 288.0553 0.000610 0 moduls 4 State 3 STATE 259.6966 259.6973 0.000682 0 moduls 5 State 3 STATE 287.6947 287.7119 0.017238 0 moduls 6 State 3 STATE 288.0546 288.0553 0.000678 0 moduls 7 State 2 STATE 259.6966 259.6972 0.000600 0 moduls 8 State 2 STATE 287.6937 287.7119 0.018168 0 moduls 9 State 2 STATE 288.0546 288.0552 0.000601 0 moduls 10 State 1 STATE 259.6966 259.6973 0.000712 0 moduls 11 State 1 STATE 287.6949 287.7119 0.017059 0 moduls 12 State 1 STATE 288.0547 288.0554 0.000707 0 moduls 13 State 0 STATE 259.6968 259.6969 0.000010 0 moduls 14 State 0 STATE 287.6883 287.7119 0.023637 0 moduls 15 State 0 STATE 288.0548 288.0548 0.000011 0 moduls #+end_example #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1200 :height 300 :session library(ggplot2); maxImbrication = df %>% .$Imbrication %>% max; plot <- NULL; dfplot <- df %>% filter(!(Value %in% c("timste", "endste"))) %>% filter(Rank < 5) %>% filter(Imbrication %in% c(0,1)); dfplot %>% ggplot() + coord_cartesian(xlim=c(startTime, endTime)) + theme_bw(base_size=12) + scale_fill_brewer(palette = "Set1") -> plot; for (i in (dfplot %>% .$Imbrication %>% unique %>% sort)){ plot <- plot + geom_rect(data=(dfplot %>% filter(Imbrication == i)), aes(fill=Value, xmin=Start, xmax=End, ymin=Rank, ymax=Rank+0.9-(Imbrication*(1/5)))); } plot; #+end_src #+RESULTS: [[file:/tmp/babel-24853m4f/figure24853J5Y.png]] *** New tracing with MPI + nsi functions #+begin_src shell :results output ssh nancy.g5k cat ./alya-bsc/Executables/unix/all.scorep #+end_src #+RESULTS: #+begin_example #SCOREP_FILE_NAMES_BEGIN #EXCLUDE * #INCLUDE */Alya.f90 #SCOREP_FILE_NAMES_END SCOREP_REGION_NAMES_BEGIN EXCLUDE * INCLUDE endste INCLUDE timste INCLUDE doiter INCLUDE moduls INCLUDE nastin INCLUDE parall INCLUDE nsi_doiter INCLUDE nsi_turnon INCLUDE nsi_parall SCOREP_REGION_NAMES_END #+end_example #+begin_src shell :results output scp nancy.g5k:./scorep_12-v1_grimoire_6_96_metis_true_v3/traces.csv.gz . #+end_src #+RESULTS: #+begin_src R :results output :session :exports both df <- read_csv("traces.csv.gz", col_names=c("Nature", "Rank", "Type", "Start", "End", "Duration", "Imbrication", "Value"), progress=FALSE); df <- df %>% mutate(Value = gsub("MPI_.*", "MPI", Value)); df; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Nature = col_character(), Rank = col_integer(), Type = col_character(), Start = col_double(), End = col_double(), Duration = col_double(), Imbrication = col_double(), Value = col_character() ) # A tibble: 2,338,005 × 8 Nature Rank Type Start End Duration Imbrication Value 1 State 95 STATE 1.770337 1.770339 2.0e-06 0 MPI 2 State 95 STATE 1.770339 1.770340 1.0e-06 0 MPI 3 State 95 STATE 1.770348 1.770353 5.0e-06 0 MPI 4 State 95 STATE 1.770357 1.770449 9.3e-05 0 MPI 5 State 95 STATE 1.770450 1.770690 2.4e-04 0 MPI 6 State 95 STATE 1.770691 1.770691 0.0e+00 0 MPI 7 State 95 STATE 1.770691 1.770691 0.0e+00 0 MPI 8 State 95 STATE 1.770692 1.770693 1.0e-06 0 MPI 9 State 95 STATE 1.770693 1.770723 3.0e-05 0 MPI 10 State 95 STATE 1.770724 1.770731 8.0e-06 0 MPI # ... with 2,337,995 more rows #+end_example #+begin_src R :results output :session :exports both startTime <- df %>% filter(Value == "timste") %>% arrange(Start) %>% slice(1) %>% .$End startTime endTime <- df %>% filter(Value == "endste") %>% arrange(End) %>% slice(n()) %>% .$End endTime maxImbrication = df %>% .$Imbrication %>% max; maxImbrication #+end_src #+RESULTS: : [1] 126.1242 : [1] 153.975 : [1] 5 #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1200 :height 300 :session library(ggplot2); plot <- NULL; dfplot <- df %>% filter(Rank != 0) %>% filter(!(Value %in% c("timste", "endste", "nsi_turnon"))) %>% filter(Imbrication != 5) %>% filter(Rank < 5); dfplot %>% ggplot() + coord_cartesian(xlim=c(startTime, endTime)) + xlim(startTime, endTime) + theme_bw(base_size=12) + scale_fill_brewer(palette = "Set1") -> plot; for (i in (dfplot %>% .$Imbrication %>% unique %>% sort)){ plot <- plot + geom_rect(data=(dfplot %>% filter(Imbrication == i)), aes(fill=Value, xmin=Start, xmax=End, ymin=Rank, ymax=Rank+0.9-(Imbrication*(1/10)))); } plot; #+end_src #+RESULTS: [[file:/tmp/babel-24853m4f/figure24853ygt.png]] Conclusions: - I should take the end =timste= and =endste= because =timste= is long - =nsi_doiter= (or the more general =doiter=) state has the calculations, but it has a lot of MPI calls within - =nsi_turnon= is not necessary #+begin_src R :results output :session :exports both df %>% filter(Rank == 4, Start > 135, End < 140) %>% filter(!(Value %in% c("timste", "endste", "doiter", "moduls"))) #+end_src #+RESULTS: #+begin_example # A tibble: 2,674 × 8 Nature Rank Type Start End Duration Imbrication Value 1 State 4 STATE 135.0003 135.0005 0.000219 4 MPI 2 State 4 STATE 135.0008 135.0010 0.000210 4 MPI 3 State 4 STATE 135.0012 135.0015 0.000228 4 MPI 4 State 4 STATE 135.0017 135.0018 0.000048 4 parall 5 State 4 STATE 135.0018 135.0018 0.000010 5 MPI 6 State 4 STATE 135.0018 135.0018 0.000003 5 MPI 7 State 4 STATE 135.0018 135.0018 0.000004 5 MPI 8 State 4 STATE 135.0018 135.0018 0.000002 5 MPI 9 State 4 STATE 135.0018 135.0018 0.000003 5 MPI 10 State 4 STATE 135.0018 135.0018 0.000001 5 MPI # ... with 2,664 more rows #+end_example #+begin_src R :results output :session :exports both df %>% filter(Imbrication == 5) %>% .$Value %>% unique #+end_src #+RESULTS: : [1] "MPI" #+begin_src R :results output :session :exports both df %>% filter(Value == "nsi_turnon") %>% filter(Rank == 4) #+end_src #+RESULTS: : # A tibble: 1 × 8 : Nature Rank Type Start End Duration Imbrication Value : : 1 State 4 STATE 125.57 125.6114 0.041432 2 nsi_turnon *** New round #+begin_src shell :results output ssh nancy.g5k cat ./alya-bsc/Executables/unix/all.scorep #+end_src #+RESULTS: #+begin_example #SCOREP_FILE_NAMES_BEGIN #EXCLUDE * #INCLUDE */Alya.f90 #SCOREP_FILE_NAMES_END SCOREP_REGION_NAMES_BEGIN EXCLUDE * # Timestep begins when rank leaves the timste state INCLUDE timste # Timestep ends when rank enters the endste INCLUDE endste # doiter is the iteration itself INCLUDE doiter # Nastin calls INCLUDE nastin INCLUDE nsi_turnon INCLUDE nsi_timste INCLUDE nsi_iniunk INCLUDE nsi_begste INCLUDE nsi_doiter INCLUDE nsi_concou INCLUDE nsi_conblk INCLUDE nsi_newmsh INCLUDE nsi_endste INCLUDE nsi_filter INCLUDE nsi_output INCLUDE nsi_turnof SCOREP_REGION_NAMES_END #+end_example #+begin_src shell :results output scp nancy.g5k:./scorep_12-v1_grimoire_6_96_metis_true_v4/traces.csv.gz . #+end_src #+RESULTS: #+begin_src R :results output :session :exports both df <- read_csv("traces.csv.gz", col_names=c("Nature", "Rank", "Type", "Start", "End", "Duration", "Imbrication", "Value"), progress=FALSE); df <- df %>% mutate(Value = gsub("MPI_.*", "MPI", Value)); startTime <- df %>% filter(Value == "timste") %>% arrange(Start) %>% slice(1) %>% .$End startTime endTime <- df %>% filter(Value == "endste") %>% arrange(End) %>% slice(n()) %>% .$End endTime maxImbrication = df %>% .$Imbrication %>% max; maxImbrication df; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Nature = col_character(), Rank = col_integer(), Type = col_character(), Start = col_double(), End = col_double(), Duration = col_double(), Imbrication = col_double(), Value = col_character() ) [1] 123.4664 [1] 151.6115 [1] 3 # A tibble: 1,906,194 × 8 Nature Rank Type Start End Duration Imbrication Value 1 State 95 STATE 1.110072 1.110075 0.000003 0 MPI 2 State 95 STATE 1.110076 1.110077 0.000001 0 MPI 3 State 95 STATE 1.110085 1.110090 0.000005 0 MPI 4 State 95 STATE 1.110091 1.110191 0.000100 0 MPI 5 State 95 STATE 1.110191 1.110424 0.000232 0 MPI 6 State 95 STATE 1.110425 1.110425 0.000000 0 MPI 7 State 95 STATE 1.110425 1.110425 0.000000 0 MPI 8 State 95 STATE 1.110426 1.110427 0.000001 0 MPI 9 State 95 STATE 1.110427 1.110463 0.000036 0 MPI 10 State 95 STATE 1.110463 1.110470 0.000007 0 MPI # ... with 1,906,184 more rows #+end_example #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1200 :height 300 :session library(ggplot2); plot <- NULL; dfplot <- df %>% filter(Rank != 0) %>% filter(!(Value %in% c("timste", "endste"))) %>% filter(Imbrication != 5) %>% filter(Rank < 5); dfplot %>% ggplot() + coord_cartesian(xlim=c(135, 145)) + #startTime, endTime)) + # xlim(startTime, endTime) + theme_bw(base_size=12) + scale_fill_brewer(palette = "Set1") -> plot; for (i in (dfplot %>% .$Imbrication %>% unique %>% sort)){ plot <- plot + geom_rect(data=(dfplot %>% filter(Imbrication == i)), aes(fill=Value, xmin=Start, xmax=End, ymin=Rank, ymax=Rank+0.9-(Imbrication*(1/10)))); } plot; #+end_src #+RESULTS: [[file:/tmp/babel-24853m4f/figure24853mXu.png]] Still not there, I can't see any function in between. *** New round with =nsi_solite= #+begin_src shell :results output ssh nancy.g5k cat ./alya-bsc/Executables/unix/all.scorep #+end_src #+RESULTS: #+begin_example #SCOREP_FILE_NAMES_BEGIN #EXCLUDE * #INCLUDE */Alya.f90 #SCOREP_FILE_NAMES_END SCOREP_REGION_NAMES_BEGIN EXCLUDE * # Timestep begins when rank leaves the timste state INCLUDE timste # Timestep ends when rank enters the endste INCLUDE endste # doiter is the iteration itself INCLUDE doiter # Nastin calls INCLUDE nastin INCLUDE nsi_turnon INCLUDE nsi_timste INCLUDE nsi_iniunk INCLUDE nsi_begste INCLUDE nsi_doiter INCLUDE nsi_concou INCLUDE nsi_conblk INCLUDE nsi_newmsh INCLUDE nsi_endste INCLUDE nsi_filter INCLUDE nsi_output INCLUDE nsi_turnof # the solver INCLUDE nsi_solite SCOREP_REGION_NAMES_END #+end_example #+begin_src shell :results output scp nancy.g5k:./scorep_12-v1_grimoire_6_96_metis_true_v5/traces.csv.gz . #+end_src #+RESULTS: #+begin_src R :results output :session :exports both df <- read_csv("traces.csv.gz", col_names=c("Nature", "Rank", "Type", "Start", "End", "Duration", "Imbrication", "Value"), progress=FALSE); df <- df %>% mutate(Value = gsub("MPI_.*", "MPI", Value)); startTime <- df %>% filter(Value == "timste") %>% arrange(Start) %>% slice(1) %>% .$End startTime endTime <- df %>% filter(Value == "endste") %>% arrange(End) %>% slice(n()) %>% .$End endTime maxImbrication = df %>% .$Imbrication %>% max; maxImbrication df; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Nature = col_character(), Rank = col_integer(), Type = col_character(), Start = col_double(), End = col_double(), Duration = col_double(), Imbrication = col_double(), Value = col_character() ) [1] 125.3257 [1] 153.2368 [1] 4 # A tibble: 1,906,674 × 8 Nature Rank Type Start End Duration Imbrication Value 1 State 95 STATE 1.025796 1.025799 0.000002 0 MPI 2 State 95 STATE 1.025800 1.025800 0.000001 0 MPI 3 State 95 STATE 1.025809 1.025814 0.000005 0 MPI 4 State 95 STATE 1.025815 1.025919 0.000104 0 MPI 5 State 95 STATE 1.025919 1.026159 0.000240 0 MPI 6 State 95 STATE 1.026160 1.026161 0.000000 0 MPI 7 State 95 STATE 1.026161 1.026161 0.000000 0 MPI 8 State 95 STATE 1.026162 1.026163 0.000001 0 MPI 9 State 95 STATE 1.026163 1.026200 0.000037 0 MPI 10 State 95 STATE 1.026200 1.026206 0.000005 0 MPI # ... with 1,906,664 more rows #+end_example #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1200 :height 300 :session library(ggplot2); plot <- NULL; dfplot <- df %>% filter(Rank != 0) %>% filter(!(Value %in% c("timste", "endste", "doiter", "nastin"))) %>% filter(Imbrication != 5) %>% filter(Rank < 5); dfplot %>% ggplot() + coord_cartesian(xlim=c(startTime, endTime)) + theme_bw(base_size=12) + scale_fill_brewer(palette = "Set1") -> plot; for (i in (dfplot %>% .$Imbrication %>% unique %>% sort)){ plot <- plot + geom_rect(data=(dfplot %>% filter(Imbrication == i)), aes(fill=Value, xmin=Start, xmax=End, ymin=Rank, ymax=Rank+0.9-(Imbrication*(1/10)))); } plot; #+end_src #+RESULTS: [[file:/tmp/babel-24853m4f/figure24853ZbQ.png]] We have five =nsi_solite= which matches: #+BEGIN_EXAMPLE --| ALYA SOLVE NASTIN (MSMSC) --| ALYA SOLVE NASTIN (MSMSC) --| ALYA SOLVE NASTIN (MSMSC) --| ALYA SOLVE NASTIN (MSMSC) --| ALYA SOLVE NASTIN (MSMSC) #+END_EXAMPLE There are still some MPI calls in between. Perhaps one of each MSMSC. *** With many functions called by nsi solite #+begin_src shell :results output ssh nancy.g5k cat ./alya-bsc/Executables/unix/all.scorep #+end_src #+RESULTS: #+begin_example #SCOREP_FILE_NAMES_BEGIN #EXCLUDE * #INCLUDE */Alya.f90 #SCOREP_FILE_NAMES_END SCOREP_REGION_NAMES_BEGIN EXCLUDE * # Timestep begins when rank leaves the timste state INCLUDE timste # Timestep ends when rank enters the endste INCLUDE endste # doiter is the iteration itself INCLUDE doiter # Nastin calls INCLUDE nastin INCLUDE nsi_turnon INCLUDE nsi_timste INCLUDE nsi_iniunk INCLUDE nsi_begste INCLUDE nsi_doiter INCLUDE nsi_concou INCLUDE nsi_conblk INCLUDE nsi_newmsh INCLUDE nsi_endste INCLUDE nsi_filter INCLUDE nsi_output INCLUDE nsi_turnof # the solver INCLUDE nsi_solite # functions called by the nsi solver INCLUDE nsi_updbcs INCLUDE nsi_ifconf INCLUDE nsi_updunk INCLUDE nsi_solsgs INCLUDE nsi_inisol INCLUDE nsi_matrix INCLUDE nsimatndof INCLUDE nsi_agmgsol INCLUDE solver INCLUDE nsi_multi_step_fs_solution INCLUDE nsi_fractional_step_solution INCLUDE nsi_schur_complement_solution INCLUDE nsi_solver_postprocess SCOREP_REGION_NAMES_END #+end_example #+begin_src shell :results output scp nancy.g5k:./scorep_12-v1_grimoire_6_96_metis_true_v6/traces.csv.gz . #+end_src #+RESULTS: #+begin_src R :results output :session :exports both df <- read_csv("traces.csv.gz", col_names=c("Nature", "Rank", "Type", "Start", "End", "Duration", "Imbrication", "Value"), progress=FALSE); df <- df %>% mutate(Value = gsub("MPI_.*", "MPI", Value)); startTime <- df %>% filter(Value == "timste") %>% arrange(Start) %>% slice(1) %>% .$End startTime endTime <- df %>% filter(Value == "endste") %>% arrange(End) %>% slice(n()) %>% .$End endTime maxImbrication = df %>% .$Imbrication %>% max; maxImbrication df; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Nature = col_character(), Rank = col_integer(), Type = col_character(), Start = col_double(), End = col_double(), Duration = col_double(), Imbrication = col_double(), Value = col_character() ) [1] 124.8219 [1] 152.4296 [1] 5 # A tibble: 1,914,065 × 8 Nature Rank Type Start End Duration Imbrication Value 1 State 95 STATE 1.014010 1.014012 0.000002 0 MPI 2 State 95 STATE 1.014013 1.014014 0.000001 0 MPI 3 State 95 STATE 1.014022 1.014027 0.000004 0 MPI 4 State 95 STATE 1.014028 1.014128 0.000100 0 MPI 5 State 95 STATE 1.014128 1.014358 0.000229 0 MPI 6 State 95 STATE 1.014359 1.014359 0.000000 0 MPI 7 State 95 STATE 1.014359 1.014359 0.000000 0 MPI 8 State 95 STATE 1.014360 1.014361 0.000001 0 MPI 9 State 95 STATE 1.014361 1.014398 0.000037 0 MPI 10 State 95 STATE 1.014398 1.014405 0.000007 0 MPI # ... with 1,914,055 more rows #+end_example #+begin_src R :results output :session :exports both df %>% select(Imbrication, Value) %>% unique %>% as.data.frame() %>% filter(Imbrication == 4) #+end_src #+RESULTS: : Imbrication Value : 1 4 nsi_updbcs : 2 4 nsi_ifconf : 3 4 nsi_updunk : 4 4 nsi_solsgs : 5 4 nsi_matrix : 6 4 solver : 7 4 MPI : 8 4 nsi_inisol #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1400 :height 300 :session library(ggplot2); plot <- NULL; dfplot <- df %>% filter(Rank != 0) %>% filter(Value %in% c(#"MPI", "solver", #"nsi_solite", "nsi_matrix", "nsi_inisol", "nsi_updunk", "nsi_updbcs", "nsi_ifconf", "nsi_updbcs")) %>% #, "nsi_matrix")) %>% # filter(!(Value %in% c("timste", "endste", "doiter", "nastin", "nsi_begste", "nsi_endste", "nsi_concou", "nsi_output"))) %>% filter(Imbrication != 5)# %>% # filter(Rank < 5); dfplot %>% ggplot() + coord_cartesian(xlim=c(startTime, endTime)) + xlim(startTime, endTime) + theme_bw(base_size=12) + scale_fill_brewer(palette = "Set1") -> plot; for (i in (dfplot %>% .$Imbrication %>% unique %>% sort)){ plot <- plot + geom_rect(data=(dfplot %>% filter(Imbrication == i)), aes(fill=Value, xmin=Start, xmax=End, ymin=Rank, ymax=Rank+0.9-(Imbrication*(1/10)))); } plot; #+end_src #+RESULTS: [[file:/tmp/babel-24853m4f/figure24853PbT.png]] So, for Nastin, computation time is: - =nsi_matrix= - =solver= - =nsi_inisol= (hardly seen) - *** Final attempt Check analysis of EXP12. *** SCOREP Filter File :ATTACH: :PROPERTIES: :Attachments: all.scorep :ID: ef6feaae-5f92-4706-a250-bde856794f47 :END: #+begin_src shell :results output scp nancy.g5k:./alya-bsc/Executables/unix/all.scorep . #+end_src #+RESULTS: ** 2017-05-04 Meeting D2.4 " *Report on Multilevel Programming and I/O optimizations" Presents: Guillaume, Rick, Lucas, Arnaud Link: https://rendez-vous.renater.fr/alya *** Introduction (Guillaume's Message) Guillaume's Message: I'm responsible for a deliverable of HPC4E, D2.4 "Report on Multilevel Programming and I/O optimizations". We are going to report our work on SFC as a mesh partitioner and I was wondering if you want to report the performance analysis you did on this as well. No need to be extensive, one page would be okay. *** Discussed Topics **** Load imbalance (and SFC/Metis) When the number of points is higher, the time spent in the "solver" function is reduce. That surprises us because it should be the contrary. The number of points correspond to the number of rows in the matrix associated to each rank. Instead, Rick told us that we should look instead to the number of entries in each row; that should directly correlate with the workload of each rank. So we should add a new row with the number of entries associated to each rank: =img/exp_13_solver_partition.png=. Load imbalance is independent of Metis/SFC. At one point, when load imbalance is perfect, we shall look again to the communication issue we have previously seen. **** The one-page report We still have 10 days to finish the delivrable. So we can check the number of entries previously mentionned before writing the report. ** 2017-05-10 Experimental session #+begin_src shell :results output oarsub -p "cluster='grisou'" -l "nodes=20,walltime=14:00:00" -r "$(date '+%Y-%m-%d 19:00:00')" -t deploy kadeploy3 --env-file /home/lschnorr/images/stretch_energy.env -f ${OAR_NODE_FILE} -k #+end_src ** 2018-01-21 Meeting @BSC during HP4E closure *** Pre-meeting notes based on e-mails **** Partition quality using adaptation strategy: run-time measurements + unity tests (Ricard) The SFC partitioning process has shown to be very fast: a mesh of 30M elements can be partitioned in 0.05 seconds using 4K CPUs. As you say, next steps must be focused on the partition quality. In particular, I’m thinking on a partition adaptation strategy based on run-time measurements or some initial measurements on unity tests. **** Different input to check for variability (Lucas) Sure. Back in those days, we have instrumented the kernels that were used in one of the simulations to measure their run-time ie. the per-iteration load of each rank. The gathered information can be used to understand how such load evolves along the execution. As you plan, perhaps we can leverage on that to improve the partition quality. I wonder if we can play with many different inputs to check for variability. **** LB with co-execution on hybrid systems where a proper load distribution is hard to guess (Ricard+Arnaud) Le fait de lancer plusieurs codes (couplés ou pas) sur les mêmes ressources. Par exemple, dans Alya, qui fait de la multi physique, on peut activer plusieurs "modules" (un pour l'atmosphère, un pour l'océan, un pour ...) qui sont en fait des programmes autonomes mais couplés. Tu peux mettre un module sur une partie des ressources, un module sur une autre partie des ressources ou les deux modules sur l'ensemble des ressources. Pour moi, quand tu les met au même endroit, c'est un exemple de co-execution et sinon, ils se partagent quand même les ressources réseau. On parle de co-execution aussi au niveau batch scheduler quand on s'autorise à exécuter deux applis vraiment indépendantes sur les mêmes ressources mais je ne pense pas que ça soit ça dont il parle. Là, c'est plusieurs modules au sein de la même application et histoire de simplifier les choses ces modules utilisent plus ou moins les GPUs, plus ou moins en même temps selon la façon dont c'est programmé.. *** Meeting - Partitioning is very fast, so we can run several times until the quality is very good. There are two approaches to improve the quality: the first one is based on using realtime measuremnets from the Alya; the second is with a unity test (mimicing the Alya code). The second has some disavatnages but is pretty much faster. - They are using a Hilbert SFC because it provides better locality. - Other approaches: Peano, Morton - The current weight: the number of elements, mass center of the element - Other approaches Ricard tried: more sofisticated approaches but it is a manual approach, very slow. A runtime approach with monitoring is better. - They read the mesh in parallel - Boxes and subboxes: the distribution based on SFC and the weight of each box is passed throught (acumulated) to other boxes. - Test cases: - sniff (17.7 M elements, 6.8M nodes); \approx1K cores (60K elements per core) - in memory occupation: 50K (1GBytes per core) - combustor (28.9 M elements, 7M nodes); \approx2K cores - 30 to 60 number of elements per core - Cartesian division from the initial global bounding box - Each rank reads the mesh in parallel and send to each box (another rank is responsible) the number of elements and their weight - Starts with a coarse grid, but a finer cartesion grid used for the SFC gives better initial LB. He does a very thin grid and its good. - How to improve the SFC. There are two options - You cut the line before if the weight is too much - Change the cartesian in some axis - Change the granularity of one box with a finer SFC - Arnaud (another approach): perturbate the partioning and collect measurements with multiple SFC configurations (with cuts slightly before and after) and then, ** 2018-01-29 Prep data for Arnaud (three stable runs) *** Introduction We are talking about experiences: - EXP24 (5 iterations) - It is a bit strange as we previously saw - See figure in to remember #+BEGIN_EXAMPLE ** 4-node grisou (64 cores + 1 core) 1its, new round *** Visualization / Analysis **** Merge the cumsum of exp24 and exp25 #+END_EXAMPLE - EXP25 (1 iteration) - EXP26 (5 iterations) *** Gather data #+name: exp24_PREP #+header: :var dep0=exp24_enrich #+header: :var dep1=exp24_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% mutate(Exp = "exp24") -> exp24.PREP #+end_src #+RESULTS: exp24_PREP : Joining, by = "Rank" #+name: exp25_PREP #+header: :var dep0=exp25_enrich #+header: :var dep1=exp25_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% mutate(Exp = "exp25") -> exp25.PREP #+end_src #+RESULTS: exp25_PREP : Joining, by = "Rank" #+name: exp26_PREP #+header: :var dep0=exp26_enrich #+header: :var dep1=exp26_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% mutate(Exp = "exp26") -> exp26.PREP #+end_src #+RESULTS: exp26_PREP : Joining, by = "Rank" #+begin_src R :results output :session :exports both exp24.PREP %>% bind_rows(exp25.PREP) %>% bind_rows(exp26.PREP) %>% select(Phase, ID, Rank, NELEM,NPOIN,NBOUN,NPOI32,Exp) -> exp.PREP exp.PREP %>% select(Phase, Exp) %>% unique #+end_src #+RESULTS: #+begin_example # A tibble: 11 x 2 Phase Exp 1 1 exp24 2 2 exp24 3 3 exp24 4 4 exp24 5 5 exp24 6 1 exp25 7 1 exp26 8 2 exp26 9 3 exp26 10 4 exp26 11 5 exp26 #+end_example *** Save the data here :ATTACH: :PROPERTIES: :Attachments: exp24-exp25-exp26.csv.gz :ID: 2c325c57-678e-491d-a7d4-f71be90dbd73 :END: #+begin_src R :results output :session :exports both write_csv(exp.PREP, "exp24-exp25-exp26.csv.gz") #+end_src #+RESULTS: ** 2018-02-06 New hypothesis Check *** PAST: ping-pong effects can be avoided by learning from the past (many rounds) Arnaud. *** GRANULARITY: a finer SFC granularity can reveal necessary details to LB 10 iterations of the test case with 128, 256, and 512 workers, for 20 refinement rounds, with granularity levels for each partitioner sub-box of 128_ip (the current def), 256_ip, and 512_ip. Please, let me know if you think that is enough. I also intend to keep CRITERIA equals to 1 (unchanged) to avoid changing everything at once, except if I can allocated another cluster for that specific test. *** CRITERIA: use a different =CRITERIA = 2_ip= so we consider weighted elements * Experiments ** 2017-03-31 Grisou experiments #+begin_src shell :results output SCRIPT=grisou-script.sh rm -f ${SCRIPT} echo 'export PATH=$PATH:$HOME/install/nova/scorep-3.0-alya/bin' >> ${SCRIPT} echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/install/nova/scorep-3.0-alya/lib' >> ${SCRIPT} for CASE in sfc metis; do for NP in 127 112 96 80 64 48 32 16; do NP=$((NP+1)) UNIQUE=taurus-np${NP}-${CASE} echo "./scripts/run_alya_experiment.sh machine-file ${NP} ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE" echo "mv $UNIQUE /home/lschnorr/" done done >> ${SCRIPT} cat ${SCRIPT} #+end_src #+RESULTS: #+begin_example export PATH=$PATH:$HOME/install/nova/scorep-3.0-alya/bin export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/install/nova/scorep-3.0-alya/lib ./scripts/run_alya_experiment.sh machine-file 128 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np128-sfc mv taurus-np128-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 113 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np113-sfc mv taurus-np113-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 97 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np97-sfc mv taurus-np97-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 81 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np81-sfc mv taurus-np81-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 65 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np65-sfc mv taurus-np65-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 49 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np49-sfc mv taurus-np49-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 33 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np33-sfc mv taurus-np33-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 17 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_sfc/fensap.dat taurus-np17-sfc mv taurus-np17-sfc /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 128 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np128-metis mv taurus-np128-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 113 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np113-metis mv taurus-np113-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 97 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np97-metis mv taurus-np97-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 81 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np81-metis mv taurus-np81-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 65 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np65-metis mv taurus-np65-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 49 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np49-metis mv taurus-np49-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 33 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np33-metis mv taurus-np33-metis /home/lschnorr/ ./scripts/run_alya_experiment.sh machine-file 17 ~/alya-bsc/Executables/unix/Alya.x ~/WORK-RICARD/resp_metis/fensap.dat taurus-np17-metis mv taurus-np17-metis /home/lschnorr/ #+end_example Those experiments have failed. ** 2017-03-31 2-node grisou debugging experiments :EXP1: Job 1224689 @ Nancy #+begin_src shell :results output git clone https://gitlab.inria.fr/schnorr/Alya-Perf.git #+end_src Requirements - The mesh: =$HOME/node_denis/= - The case study: =$HOME/WORK-RICARD/= - Scorep 3.0 installed at: =$HOME/install/nova/scorep-3.0-alya/= - Alya: =$HOME/alya-bsc/Executables/unix/Alya.x= - Instrumented with the aid of scorep #+BEGIN_EXAMPLE SCOREP=scorep --compiler --instrument-filter=/home/lschnorr/alya-bsc/Executables/unix/alya-timestep-filters.scorep #+END_EXAMPLE Contents of the filter #+BEGIN_EXAMPLE SCOREP_REGION_NAMES_BEGIN EXCLUDE * INCLUDE endste INCLUDE timste SCOREP_REGION_NAMES_END #+END_EXAMPLE Experimental design #+begin_src shell :results output CLUSTER=grisou NODES=2 MACHINEFILE=machine-file SCRIPT="script_${CLUSTER}_${NODES}_nodes.sh" rm -f ${SCRIPT} for NP in 31 16; do for CASE in sfc metis; do NPN=$((NP+1)) UNIQUE=${CLUSTER}_${NODES}_${NPN}_${CASE} echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE" echo "mv $UNIQUE.dir /home/lschnorr/" done done >> ${SCRIPT} chmod 755 ${SCRIPT} cat ${SCRIPT} #+end_src #+RESULTS: : ./scripts/run_alya_experiment.sh machine-file 32 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat grisou_2_32_sfc : mv grisou_2_32_sfc.dir /home/lschnorr/ : ./scripts/run_alya_experiment.sh machine-file 32 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat grisou_2_32_metis : mv grisou_2_32_metis.dir /home/lschnorr/ : ./scripts/run_alya_experiment.sh machine-file 17 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat grisou_2_17_sfc : mv grisou_2_17_sfc.dir /home/lschnorr/ : ./scripts/run_alya_experiment.sh machine-file 17 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat grisou_2_17_metis : mv grisou_2_17_metis.dir /home/lschnorr/ ** 2017-03-31 10-node grisou debugging experiments :EXP2:EXP3: Same as previous entry, but with 10 nodes. Job 1224741 @ Nancy Kadeploy the image: =/home/lschnorr/images/stretch_energy_v6_2017_03_30.tgz= Same requirements as previous entry. Experimental design #+begin_src shell :results output EID=3 CLUSTER=grisou NODES=10 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ SCRIPT="script_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design for NP in 159 144 128 112 96; do for CASE in sfc metis; do NPN=$((NP+1)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE} echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE" echo "cp -prfv $UNIQUE.dir ${EDIR}" done done >> ${SCRIPT} echo "mv $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_3_grisou_10 ]; then echo "Directory/File '/home/lschnorr//exp_3_grisou_10' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 160 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 3_grisou_10_160_sfc cp -prfv 3_grisou_10_160_sfc.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 160 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 3_grisou_10_160_metis cp -prfv 3_grisou_10_160_metis.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 145 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 3_grisou_10_145_sfc cp -prfv 3_grisou_10_145_sfc.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 145 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 3_grisou_10_145_metis cp -prfv 3_grisou_10_145_metis.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 129 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 3_grisou_10_129_sfc cp -prfv 3_grisou_10_129_sfc.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 129 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 3_grisou_10_129_metis cp -prfv 3_grisou_10_129_metis.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 113 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 3_grisou_10_113_sfc cp -prfv 3_grisou_10_113_sfc.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 113 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 3_grisou_10_113_metis cp -prfv 3_grisou_10_113_metis.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 97 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 3_grisou_10_97_sfc cp -prfv 3_grisou_10_97_sfc.dir /home/lschnorr//exp_3_grisou_10 ./scripts/run_alya_experiment.sh machine-file 97 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 3_grisou_10_97_metis cp -prfv 3_grisou_10_97_metis.dir /home/lschnorr//exp_3_grisou_10 mv script_grisou_10_nodes.sh /home/lschnorr//exp_3_grisou_10 This script is at 'script_grisou_10_nodes.sh', commit it. #+end_example Faster way to copy is to tar (without compression) all files, and then copy using (not the g5kg) #+BEGIN_EXAMPLE rsync --progress grisou-51.nancy.g5kg:~/exp_2_grisou_10.tar . #+END_EXAMPLE Two replications have been made: - exp 2 - exp 3 ** 2017-04-03 23-node (368 cores) nova in April 3rd :EXP4: *** Preparation for nova (prep-4-v[12]) #+begin_src shell :results output EID=prep-4 CLUSTER=grisou NODES=4 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design for NP in 62 48 32 17; do for CASE in sfc metis; do NPN=$((NP+1)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE} echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE" echo "cp -prfv $UNIQUE.dir ${EDIR}" done done >> ${SCRIPT} echo "mv $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_prep-4_grisou_4 ]; then echo "Directory/File '/home/lschnorr//exp_prep-4_grisou_4' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_prep-4_grisou_4 ./scripts/run_alya_experiment.sh machine-file 63 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat prep-4_grisou_4_63_sfc cp -prfv prep-4_grisou_4_63_sfc.dir /home/lschnorr//exp_prep-4_grisou_4 ./scripts/run_alya_experiment.sh machine-file 63 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat prep-4_grisou_4_63_metis cp -prfv prep-4_grisou_4_63_metis.dir /home/lschnorr//exp_prep-4_grisou_4 ./scripts/run_alya_experiment.sh machine-file 49 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat prep-4_grisou_4_49_sfc cp -prfv prep-4_grisou_4_49_sfc.dir /home/lschnorr//exp_prep-4_grisou_4 ./scripts/run_alya_experiment.sh machine-file 49 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat prep-4_grisou_4_49_metis cp -prfv prep-4_grisou_4_49_metis.dir /home/lschnorr//exp_prep-4_grisou_4 ./scripts/run_alya_experiment.sh machine-file 33 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat prep-4_grisou_4_33_sfc cp -prfv prep-4_grisou_4_33_sfc.dir /home/lschnorr//exp_prep-4_grisou_4 ./scripts/run_alya_experiment.sh machine-file 33 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat prep-4_grisou_4_33_metis cp -prfv prep-4_grisou_4_33_metis.dir /home/lschnorr//exp_prep-4_grisou_4 ./scripts/run_alya_experiment.sh machine-file 18 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat prep-4_grisou_4_18_sfc cp -prfv prep-4_grisou_4_18_sfc.dir /home/lschnorr//exp_prep-4_grisou_4 ./scripts/run_alya_experiment.sh machine-file 18 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat prep-4_grisou_4_18_metis cp -prfv prep-4_grisou_4_18_metis.dir /home/lschnorr//exp_prep-4_grisou_4 mv script_prep-4_grisou_4_nodes.sh /home/lschnorr//exp_prep-4_grisou_4 This script is at 'script_prep-4_grisou_4_nodes.sh', commit it. #+end_example *** Full 23-node nova JobId 854496 @ Lyon. #+begin_src shell :results output EID=4 CLUSTER=grisou NODES=23 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design for NP in 367 352 336 272 208 144 80 16; do for CASE in sfc metis; do NPN=$((NP+1)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE} echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE" echo "cp -prfv $UNIQUE.dir ${EDIR}" done done >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_4_grisou_23 ]; then echo "Directory/File '/home/lschnorr//exp_4_grisou_23' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 368 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 4_grisou_23_368_sfc cp -prfv 4_grisou_23_368_sfc.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 368 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 4_grisou_23_368_metis cp -prfv 4_grisou_23_368_metis.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 353 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 4_grisou_23_353_sfc cp -prfv 4_grisou_23_353_sfc.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 353 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 4_grisou_23_353_metis cp -prfv 4_grisou_23_353_metis.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 337 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 4_grisou_23_337_sfc cp -prfv 4_grisou_23_337_sfc.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 337 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 4_grisou_23_337_metis cp -prfv 4_grisou_23_337_metis.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 273 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 4_grisou_23_273_sfc cp -prfv 4_grisou_23_273_sfc.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 273 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 4_grisou_23_273_metis cp -prfv 4_grisou_23_273_metis.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 209 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 4_grisou_23_209_sfc cp -prfv 4_grisou_23_209_sfc.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 209 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 4_grisou_23_209_metis cp -prfv 4_grisou_23_209_metis.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 145 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 4_grisou_23_145_sfc cp -prfv 4_grisou_23_145_sfc.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 145 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 4_grisou_23_145_metis cp -prfv 4_grisou_23_145_metis.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 81 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 4_grisou_23_81_sfc cp -prfv 4_grisou_23_81_sfc.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 81 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 4_grisou_23_81_metis cp -prfv 4_grisou_23_81_metis.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 17 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 4_grisou_23_17_sfc cp -prfv 4_grisou_23_17_sfc.dir /home/lschnorr//exp_4_grisou_23 ./scripts/run_alya_experiment.sh machine-file 17 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 4_grisou_23_17_metis cp -prfv 4_grisou_23_17_metis.dir /home/lschnorr//exp_4_grisou_23 cp script_4_grisou_23_nodes.sh /home/lschnorr//exp_4_grisou_23 This script is at 'script_4_grisou_23_nodes.sh', commit it. #+end_example ** 2017-04-03 50-node (800 cores) grisou in April 4th :EXP5: #+begin_src shell :results output EID=5 VERSION=1 CLUSTER=grisou NODES=50 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ SCRIPT="script_${EID}_v${VERSION}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}-v${VERSION}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design for NP in 800 720 640 560 480 400; do for CASE in sfc metis; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE} echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE false" echo "cp -prfv $UNIQUE.dir ${EDIR}" done done >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_5-v1_grisou_50 ]; then echo "Directory/File '/home/lschnorr//exp_5-v1_grisou_50' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 800 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 5_grisou_50_800_sfc false cp -prfv 5_grisou_50_800_sfc.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 800 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 5_grisou_50_800_metis false cp -prfv 5_grisou_50_800_metis.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 720 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 5_grisou_50_720_sfc false cp -prfv 5_grisou_50_720_sfc.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 720 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 5_grisou_50_720_metis false cp -prfv 5_grisou_50_720_metis.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 640 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 5_grisou_50_640_sfc false cp -prfv 5_grisou_50_640_sfc.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 640 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 5_grisou_50_640_metis false cp -prfv 5_grisou_50_640_metis.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 560 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 5_grisou_50_560_sfc false cp -prfv 5_grisou_50_560_sfc.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 560 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 5_grisou_50_560_metis false cp -prfv 5_grisou_50_560_metis.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 480 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 5_grisou_50_480_sfc false cp -prfv 5_grisou_50_480_sfc.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 480 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 5_grisou_50_480_metis false cp -prfv 5_grisou_50_480_metis.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 400 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 5_grisou_50_400_sfc false cp -prfv 5_grisou_50_400_sfc.dir /home/lschnorr//exp_5-v1_grisou_50 ./scripts/run_alya_experiment.sh machine-file 400 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 5_grisou_50_400_metis false cp -prfv 5_grisou_50_400_metis.dir /home/lschnorr//exp_5-v1_grisou_50 cp script_5_v1_grisou_50_nodes.sh /home/lschnorr//exp_5-v1_grisou_50 This script is at 'script_5_v1_grisou_50_nodes.sh', commit it. #+end_example ** 2017-04-03 7-node (112 cores) grimoire with Infiniband :EXP6: #+begin_src shell :results output EID=6 CLUSTER=grimoire NODES=7 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design for NP in 111 96; do for CASE in sfc metis; do NPN=$((NP+1)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE} echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE true" echo "cp -prfv $UNIQUE.dir ${EDIR}" done done >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_6_grimoire_7 ]; then echo "Directory/File '/home/lschnorr//exp_6_grimoire_7' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_6_grimoire_7 ./scripts/run_alya_experiment.sh machine-file 112 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 6_grimoire_7_112_sfc true cp -prfv 6_grimoire_7_112_sfc.dir /home/lschnorr//exp_6_grimoire_7 ./scripts/run_alya_experiment.sh machine-file 112 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 6_grimoire_7_112_metis true cp -prfv 6_grimoire_7_112_metis.dir /home/lschnorr//exp_6_grimoire_7 ./scripts/run_alya_experiment.sh machine-file 97 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 6_grimoire_7_97_sfc true cp -prfv 6_grimoire_7_97_sfc.dir /home/lschnorr//exp_6_grimoire_7 ./scripts/run_alya_experiment.sh machine-file 97 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 6_grimoire_7_97_metis true cp -prfv 6_grimoire_7_97_metis.dir /home/lschnorr//exp_6_grimoire_7 cp script_6_grimoire_7_nodes.sh /home/lschnorr//exp_6_grimoire_7 This script is at 'script_6_grimoire_7_nodes.sh', commit it. #+end_example ** 2017-04-04 4-node (64 cores) grimoire with Infiniband :EXP7: The goal is simply confirm experimentally that OpenMPI is doing openib communications instead of ethernet when told so. #+begin_src shell :results output EID=7 CLUSTER=grimoire NODES=4 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design NPN=112 for INFINIBAND in true false; do for CASE in sfc metis; do UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE} echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" done done >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_7_grimoire_4 ]; then echo "Directory/File '/home/lschnorr//exp_7_grimoire_4' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_7_grimoire_4 ./scripts/run_alya_experiment.sh machine-file 112 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 7_grimoire_4_112_sfc true cp -prfv 7_grimoire_4_112_sfc.dir /home/lschnorr//exp_7_grimoire_4 ./scripts/run_alya_experiment.sh machine-file 112 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 7_grimoire_4_112_metis true cp -prfv 7_grimoire_4_112_metis.dir /home/lschnorr//exp_7_grimoire_4 ./scripts/run_alya_experiment.sh machine-file 112 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 7_grimoire_4_112_sfc false cp -prfv 7_grimoire_4_112_sfc.dir /home/lschnorr//exp_7_grimoire_4 ./scripts/run_alya_experiment.sh machine-file 112 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 7_grimoire_4_112_metis false cp -prfv 7_grimoire_4_112_metis.dir /home/lschnorr//exp_7_grimoire_4 cp script_7_grimoire_4_nodes.sh /home/lschnorr//exp_7_grimoire_4 This script is at 'script_7_grimoire_4_nodes.sh', commit it. #+end_example ** 2017-04-05 2-node (32 cores) and 4-node (64 cores) grimoire to test Infiniband :EXP8: #+begin_src shell :results output EID=8 VERSION=4 CLUSTER=grimoire NODES=4 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ EID=${EID}-v${VERSION} SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design NP=60 for INFINIBAND in true false; do for CASE in metis sfc; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_8-v4_grimoire_4 ]; then echo "Directory/File '/home/lschnorr//exp_8-v4_grimoire_4' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_8-v4_grimoire_4 # # Experiment 8-v4_grimoire_4_60_metis_true # ./scripts/run_alya_experiment.sh machine-file 60 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 8-v4_grimoire_4_60_metis_true true cp -prfv 8-v4_grimoire_4_60_metis_true.dir /home/lschnorr//exp_8-v4_grimoire_4 rm -rf 8-v4_grimoire_4_60_metis_true.dir # # Experiment 8-v4_grimoire_4_60_sfc_true # ./scripts/run_alya_experiment.sh machine-file 60 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 8-v4_grimoire_4_60_sfc_true true cp -prfv 8-v4_grimoire_4_60_sfc_true.dir /home/lschnorr//exp_8-v4_grimoire_4 rm -rf 8-v4_grimoire_4_60_sfc_true.dir # # Experiment 8-v4_grimoire_4_60_metis_false # ./scripts/run_alya_experiment.sh machine-file 60 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 8-v4_grimoire_4_60_metis_false false cp -prfv 8-v4_grimoire_4_60_metis_false.dir /home/lschnorr//exp_8-v4_grimoire_4 rm -rf 8-v4_grimoire_4_60_metis_false.dir # # Experiment 8-v4_grimoire_4_60_sfc_false # ./scripts/run_alya_experiment.sh machine-file 60 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 8-v4_grimoire_4_60_sfc_false false cp -prfv 8-v4_grimoire_4_60_sfc_false.dir /home/lschnorr//exp_8-v4_grimoire_4 rm -rf 8-v4_grimoire_4_60_sfc_false.dir # # Final # cp script_8-v4_grimoire_4_nodes.sh /home/lschnorr//exp_8-v4_grimoire_4 This script is at 'script_8-v4_grimoire_4_nodes.sh', commit it. #+end_example ** 2017-04-05 6-node grimoire with Infiniband :EXP9: #+begin_src shell :results output EID=9 VERSION=1 CLUSTER=grimoire NODES=6 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ EID=${EID}-v${VERSION} SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design NP=96 for INFINIBAND in true false; do for CASE in metis sfc; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_9-v1_grimoire_6 ]; then echo "Directory/File '/home/lschnorr//exp_9-v1_grimoire_6' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_9-v1_grimoire_6 # # Experiment 9-v1_grimoire_6_96_metis_true # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 9-v1_grimoire_6_96_metis_true true cp -prfv 9-v1_grimoire_6_96_metis_true.dir /home/lschnorr//exp_9-v1_grimoire_6 rm -rf 9-v1_grimoire_6_96_metis_true.dir # # Experiment 9-v1_grimoire_6_96_sfc_true # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 9-v1_grimoire_6_96_sfc_true true cp -prfv 9-v1_grimoire_6_96_sfc_true.dir /home/lschnorr//exp_9-v1_grimoire_6 rm -rf 9-v1_grimoire_6_96_sfc_true.dir # # Experiment 9-v1_grimoire_6_96_metis_false # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 9-v1_grimoire_6_96_metis_false false cp -prfv 9-v1_grimoire_6_96_metis_false.dir /home/lschnorr//exp_9-v1_grimoire_6 rm -rf 9-v1_grimoire_6_96_metis_false.dir # # Experiment 9-v1_grimoire_6_96_sfc_false # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 9-v1_grimoire_6_96_sfc_false false cp -prfv 9-v1_grimoire_6_96_sfc_false.dir /home/lschnorr//exp_9-v1_grimoire_6 rm -rf 9-v1_grimoire_6_96_sfc_false.dir # # Final # cp script_9-v1_grimoire_6_nodes.sh /home/lschnorr//exp_9-v1_grimoire_6 This script is at 'script_9-v1_grimoire_6_nodes.sh', commit it. #+end_example ** 2017-04-06 8-node grimoire infiniband-only after Alya modifications :EXP10: #+begin_src shell :results output EID=10 VERSION=1 CLUSTER=grimoire NODES=8 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ EID=${EID}-v${VERSION} SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design NP=128 for INFINIBAND in true; do for CASE in metis sfc; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_10-v1_grimoire_8 ]; then echo "Directory/File '/home/lschnorr//exp_10-v1_grimoire_8' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_10-v1_grimoire_8 # # Experiment 10-v1_grimoire_8_128_metis_true # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 10-v1_grimoire_8_128_metis_true true cp -prfv 10-v1_grimoire_8_128_metis_true.dir /home/lschnorr//exp_10-v1_grimoire_8 rm -rf 10-v1_grimoire_8_128_metis_true.dir # # Experiment 10-v1_grimoire_8_128_sfc_true # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 10-v1_grimoire_8_128_sfc_true true cp -prfv 10-v1_grimoire_8_128_sfc_true.dir /home/lschnorr//exp_10-v1_grimoire_8 rm -rf 10-v1_grimoire_8_128_sfc_true.dir # # Final # cp script_10-v1_grimoire_8_nodes.sh /home/lschnorr//exp_10-v1_grimoire_8 This script is at 'script_10-v1_grimoire_8_nodes.sh', commit it. #+end_example ** 2017-04-06 8-node grimoire infiniband-only without Alya modifications :EXP11: I will compile Alya before the partitioning weight modifications. #+begin_src shell :results output EID=11 VERSION=1 CLUSTER=grimoire NODES=8 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ EID=${EID}-v${VERSION} SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design NP=128 for INFINIBAND in true; do for CASE in metis sfc; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_11-v1_grimoire_8 ]; then echo "Directory/File '/home/lschnorr//exp_11-v1_grimoire_8' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_11-v1_grimoire_8 # # Experiment 11-v1_grimoire_8_128_metis_true # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 11-v1_grimoire_8_128_metis_true true cp -prfv 11-v1_grimoire_8_128_metis_true.dir /home/lschnorr//exp_11-v1_grimoire_8 rm -rf 11-v1_grimoire_8_128_metis_true.dir # # Experiment 11-v1_grimoire_8_128_sfc_true # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 11-v1_grimoire_8_128_sfc_true true cp -prfv 11-v1_grimoire_8_128_sfc_true.dir /home/lschnorr//exp_11-v1_grimoire_8 rm -rf 11-v1_grimoire_8_128_sfc_true.dir # # Final # cp script_11-v1_grimoire_8_nodes.sh /home/lschnorr//exp_11-v1_grimoire_8 This script is at 'script_11-v1_grimoire_8_nodes.sh', commit it. #+end_example ** 2017-04-07 6-node grimoire (no Alya modifications, full computing instrumentations) :EXP12: For this experiment, we shall: - remove MPI instrumentation completely - adopt compiler instrumentation to obtain some application functions #+begin_src shell :results output EID=12 VERSION=1 CLUSTER=grimoire NODES=6 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ EID=${EID}-v${VERSION} SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design for NP in 96 48; do for INFINIBAND in true; do for CASE in metis sfc; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done done >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_12-v1_grimoire_6 ]; then echo "Directory/File '/home/lschnorr//exp_12-v1_grimoire_6' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_12-v1_grimoire_6 # # Experiment 12-v1_grimoire_6_96_metis_true # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 12-v1_grimoire_6_96_metis_true true cp -prfv 12-v1_grimoire_6_96_metis_true.dir /home/lschnorr//exp_12-v1_grimoire_6 rm -rf 12-v1_grimoire_6_96_metis_true.dir # # Experiment 12-v1_grimoire_6_96_sfc_true # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 12-v1_grimoire_6_96_sfc_true true cp -prfv 12-v1_grimoire_6_96_sfc_true.dir /home/lschnorr//exp_12-v1_grimoire_6 rm -rf 12-v1_grimoire_6_96_sfc_true.dir # # Experiment 12-v1_grimoire_6_48_metis_true # ./scripts/run_alya_experiment.sh machine-file 48 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 12-v1_grimoire_6_48_metis_true true cp -prfv 12-v1_grimoire_6_48_metis_true.dir /home/lschnorr//exp_12-v1_grimoire_6 rm -rf 12-v1_grimoire_6_48_metis_true.dir # # Experiment 12-v1_grimoire_6_48_sfc_true # ./scripts/run_alya_experiment.sh machine-file 48 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 12-v1_grimoire_6_48_sfc_true true cp -prfv 12-v1_grimoire_6_48_sfc_true.dir /home/lschnorr//exp_12-v1_grimoire_6 rm -rf 12-v1_grimoire_6_48_sfc_true.dir # # Final # cp script_12-v1_grimoire_6_nodes.sh /home/lschnorr//exp_12-v1_grimoire_6 This script is at 'script_12-v1_grimoire_6_nodes.sh', commit it. #+end_example ** 2017-04-10 12-node grisou (computing instrumentations, Alya modifs yes/no) :EXP13: #+begin_src shell :results output EID=13 VERSION=1 CLUSTER=grisou NODES=12 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ EID=${EID}-v${VERSION} SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design ALYAO=Alya.x ALYAM=Alya.x.modif for NP in 192 96; do for INFINIBAND in true; do for CASE in metis sfc; do for ALYA in $ALYAO $ALYAM; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND}_${ALYA} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} /home/lschnorr/alya-bsc/Executables/unix/$ALYA /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done done >> ${SCRIPT} done echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_13-v1_grisou_12 ]; then echo "Directory/File '/home/lschnorr//exp_13-v1_grisou_12' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_13-v1_grisou_12 # # Experiment 13-v1_grisou_12_192_metis_true_Alya.x # ./scripts/run_alya_experiment.sh machine-file 192 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 13-v1_grisou_12_192_metis_true_Alya.x true cp -prfv 13-v1_grisou_12_192_metis_true_Alya.x.dir /home/lschnorr//exp_13-v1_grisou_12 rm -rf 13-v1_grisou_12_192_metis_true_Alya.x.dir # # Experiment 13-v1_grisou_12_192_metis_true_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 192 /home/lschnorr/alya-bsc/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 13-v1_grisou_12_192_metis_true_Alya.x.modif true cp -prfv 13-v1_grisou_12_192_metis_true_Alya.x.modif.dir /home/lschnorr//exp_13-v1_grisou_12 rm -rf 13-v1_grisou_12_192_metis_true_Alya.x.modif.dir # # Experiment 13-v1_grisou_12_192_sfc_true_Alya.x # ./scripts/run_alya_experiment.sh machine-file 192 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 13-v1_grisou_12_192_sfc_true_Alya.x true cp -prfv 13-v1_grisou_12_192_sfc_true_Alya.x.dir /home/lschnorr//exp_13-v1_grisou_12 rm -rf 13-v1_grisou_12_192_sfc_true_Alya.x.dir # # Experiment 13-v1_grisou_12_192_sfc_true_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 192 /home/lschnorr/alya-bsc/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 13-v1_grisou_12_192_sfc_true_Alya.x.modif true cp -prfv 13-v1_grisou_12_192_sfc_true_Alya.x.modif.dir /home/lschnorr//exp_13-v1_grisou_12 rm -rf 13-v1_grisou_12_192_sfc_true_Alya.x.modif.dir # # Experiment 13-v1_grisou_12_96_metis_true_Alya.x # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 13-v1_grisou_12_96_metis_true_Alya.x true cp -prfv 13-v1_grisou_12_96_metis_true_Alya.x.dir /home/lschnorr//exp_13-v1_grisou_12 rm -rf 13-v1_grisou_12_96_metis_true_Alya.x.dir # # Experiment 13-v1_grisou_12_96_metis_true_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 13-v1_grisou_12_96_metis_true_Alya.x.modif true cp -prfv 13-v1_grisou_12_96_metis_true_Alya.x.modif.dir /home/lschnorr//exp_13-v1_grisou_12 rm -rf 13-v1_grisou_12_96_metis_true_Alya.x.modif.dir # # Experiment 13-v1_grisou_12_96_sfc_true_Alya.x # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 13-v1_grisou_12_96_sfc_true_Alya.x true cp -prfv 13-v1_grisou_12_96_sfc_true_Alya.x.dir /home/lschnorr//exp_13-v1_grisou_12 rm -rf 13-v1_grisou_12_96_sfc_true_Alya.x.dir # # Experiment 13-v1_grisou_12_96_sfc_true_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 96 /home/lschnorr/alya-bsc/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 13-v1_grisou_12_96_sfc_true_Alya.x.modif true cp -prfv 13-v1_grisou_12_96_sfc_true_Alya.x.modif.dir /home/lschnorr//exp_13-v1_grisou_12 rm -rf 13-v1_grisou_12_96_sfc_true_Alya.x.modif.dir # # Final # cp script_13-v1_grisou_12_nodes.sh /home/lschnorr//exp_13-v1_grisou_12 This script is at 'script_13-v1_grisou_12_nodes.sh', commit it. #+end_example ** 2017-05-10 20-node grisou :EXP14: *** Introduction :ATTACH: :PROPERTIES: :Attachments: all.scorep :ID: b51a80da-07e4-4d58-844c-a73009092eaa :END: The attached file, =all.scorep=, contains the functions that are instrumented during compilation by scorep. Those functions are non-MPI codes that represent the computation load. I am using Trunk.6515.Alya, unmodified, to get =Alya.x.orig=. Then, I manually modify Alya the way described here: [[*2017-04-06 Alya modifications to get more info, set balance to 1][2017-04-06 Alya modifications to get more info, set balance to 1]] to get another binary called =Alya.x.modif= (with elements weight set to 1). It seems that the latest version of Alya (revision 6515 at least) has a slightly different code to deal with element weight (with the =iweig= variable). So, for the =modif= version, I am forcing the =iweig= to be =1_rp=, removing the =if= that is now there. #+BEGIN_EXAMPLE lschnorr@grisou-1:~/Trunk.6515.Alya/Executables/unix$ svn diff ../../Sources/kernel/parall/mod_par_partit_sfc.f90 Index: ../../Sources/kernel/parall/mod_par_partit_sfc.f90 =================================================================== --- ../../Sources/kernel/parall/mod_par_partit_sfc.f90 (revision 6515) +++ ../../Sources/kernel/parall/mod_par_partit_sfc.f90 (working copy) @@ -459,6 +459,7 @@ else iweig = 1_rp endif + iweig = 1_rp if( PAR_MY_SFC_RANK_WM /= iboxc-1 )then !if true element must be sent if(bufwei(iboxl*2) == 0_rp) then #+END_EXAMPLE Note that the file =Sources/services/parall/par_outinf.f90= needs to be modified to output per-rank weights, points, etc. This is fundamental to conduct our analysis. Here's the diff: #+BEGIN_EXAMPLE lschnorr@grisou-1:~/Trunk.6515.Alya/Executables/unix$ svn diff ../../Sources/services/parall/par_outinf.f90 Index: ../../Sources/services/parall/par_outinf.f90 =================================================================== --- ../../Sources/services/parall/par_outinf.f90 (revision 6515) +++ ../../Sources/services/parall/par_outinf.f90 (working copy) @@ -15,6 +15,7 @@ use mod_parall implicit none integer(ip) :: isubd,ksmin(2),ksmax(2),kaver,koutp,nb + integer, parameter :: out_unit_npoin=20 if( IMASTER ) then @@ -27,7 +28,11 @@ ksmin = huge(1_ip) ksmax = -huge(1_ip) kaver = 0 + ! LUCAS: manual instrumentation starts here, all code marked by LUCAS + open (unit=out_unit_npoin,file="results_NPOIN_NELEM_NELEW_NBOUN.log",action="write",status="replace") do isubd=1,npart_par + ! LUCAS: Here I need to print: npoin_par(isubd) + write (out_unit_npoin,*) "Rank ", isubd ," NPOIN ", npoin_par(isubd), " NELEM ", nelem_par(isubd), " NELEW ", nelew_par(isubd), " NBOUN ", nboun_par(isubd), " NNEIG ", lneig_par(isubd), " NBBOU ", npoin_par(isubd) if(npoin_par(isubd)> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design ALYAO=Alya.x.orig ALYAM=Alya.x.modif ALYAPATH=/home/lschnorr/Trunk.6515.Alya for NP in 320; do for INFINIBAND in false; do for CASE in metis sfc; do for ALYA in $ALYAO $ALYAM; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND}_${ALYA} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} ${ALYAPATH}/Executables/unix/$ALYA /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done done >> ${SCRIPT} done echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_14-v1_grisou_20 ]; then echo "Directory/File '/home/lschnorr//exp_14-v1_grisou_20' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_14-v1_grisou_20 # # Experiment 14-v1_grisou_20_320_metis_false_Alya.x.orig # ./scripts/run_alya_experiment.sh machine-file 320 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.orig /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 14-v1_grisou_20_320_metis_false_Alya.x.orig false cp -prfv 14-v1_grisou_20_320_metis_false_Alya.x.orig.dir /home/lschnorr//exp_14-v1_grisou_20 rm -rf 14-v1_grisou_20_320_metis_false_Alya.x.orig.dir # # Experiment 14-v1_grisou_20_320_metis_false_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 320 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 14-v1_grisou_20_320_metis_false_Alya.x.modif false cp -prfv 14-v1_grisou_20_320_metis_false_Alya.x.modif.dir /home/lschnorr//exp_14-v1_grisou_20 rm -rf 14-v1_grisou_20_320_metis_false_Alya.x.modif.dir # # Experiment 14-v1_grisou_20_320_sfc_false_Alya.x.orig # ./scripts/run_alya_experiment.sh machine-file 320 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.orig /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 14-v1_grisou_20_320_sfc_false_Alya.x.orig false cp -prfv 14-v1_grisou_20_320_sfc_false_Alya.x.orig.dir /home/lschnorr//exp_14-v1_grisou_20 rm -rf 14-v1_grisou_20_320_sfc_false_Alya.x.orig.dir # # Experiment 14-v1_grisou_20_320_sfc_false_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 320 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 14-v1_grisou_20_320_sfc_false_Alya.x.modif false cp -prfv 14-v1_grisou_20_320_sfc_false_Alya.x.modif.dir /home/lschnorr//exp_14-v1_grisou_20 rm -rf 14-v1_grisou_20_320_sfc_false_Alya.x.modif.dir # # Final # cp script_14-v1_grisou_20_nodes.sh /home/lschnorr//exp_14-v1_grisou_20 This script is at 'script_14-v1_grisou_20_nodes.sh', commit it. #+end_example ** 2017-05-11 44-node grisou (same as previous entry) :EXP15: *** Design #+begin_src shell :results output EID=15 VERSION=1 CLUSTER=grisou NODES=44 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ EID=${EID}-v${VERSION} SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design ALYAO=Alya.x.orig ALYAM=Alya.x.modif ALYAPATH=/home/lschnorr/Trunk.6515.Alya for NP in 704; do for INFINIBAND in false; do for CASE in metis sfc; do for ALYA in $ALYAO $ALYAM; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND}_${ALYA} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} ${ALYAPATH}/Executables/unix/$ALYA /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done done >> ${SCRIPT} done echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_15-v1_grisou_44 ]; then echo "Directory/File '/home/lschnorr//exp_15-v1_grisou_44' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_15-v1_grisou_44 # # Experiment 15-v1_grisou_44_704_metis_false_Alya.x.orig # ./scripts/run_alya_experiment.sh machine-file 704 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.orig /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 15-v1_grisou_44_704_metis_false_Alya.x.orig false cp -prfv 15-v1_grisou_44_704_metis_false_Alya.x.orig.dir /home/lschnorr//exp_15-v1_grisou_44 rm -rf 15-v1_grisou_44_704_metis_false_Alya.x.orig.dir # # Experiment 15-v1_grisou_44_704_metis_false_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 704 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 15-v1_grisou_44_704_metis_false_Alya.x.modif false cp -prfv 15-v1_grisou_44_704_metis_false_Alya.x.modif.dir /home/lschnorr//exp_15-v1_grisou_44 rm -rf 15-v1_grisou_44_704_metis_false_Alya.x.modif.dir # # Experiment 15-v1_grisou_44_704_sfc_false_Alya.x.orig # ./scripts/run_alya_experiment.sh machine-file 704 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.orig /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 15-v1_grisou_44_704_sfc_false_Alya.x.orig false cp -prfv 15-v1_grisou_44_704_sfc_false_Alya.x.orig.dir /home/lschnorr//exp_15-v1_grisou_44 rm -rf 15-v1_grisou_44_704_sfc_false_Alya.x.orig.dir # # Experiment 15-v1_grisou_44_704_sfc_false_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 704 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 15-v1_grisou_44_704_sfc_false_Alya.x.modif false cp -prfv 15-v1_grisou_44_704_sfc_false_Alya.x.modif.dir /home/lschnorr//exp_15-v1_grisou_44 rm -rf 15-v1_grisou_44_704_sfc_false_Alya.x.modif.dir # # Final # cp script_15-v1_grisou_44_nodes.sh /home/lschnorr//exp_15-v1_grisou_44 This script is at 'script_15-v1_grisou_44_nodes.sh', commit it. #+end_example ** 2017-05-11 8-node grimoire (as previous, Infiniband and Ethernet) :EXP16: *** Design #+begin_src shell :results output EID=16 VERSION=1 CLUSTER=grimoire NODES=8 MACHINEFILE=machine-file G5KHOME=/home/lschnorr/ EID=${EID}-v${VERSION} SCRIPT="script_${EID}_${CLUSTER}_${NODES}_nodes.sh" rm -f $SCRIPT EDIR="$G5KHOME/exp_${EID}_${CLUSTER}_${NODES}" echo "if [ -e "$EDIR" ]; then" >> $SCRIPT echo " echo \"Directory/File '${EDIR}' already exists. Remove it or rename it.\"" >> $SCRIPT echo " exit" >> $SCRIPT echo "fi" >> $SCRIPT # Create experimental directory in the home echo "mkdir -p $EDIR" >> $SCRIPT # The design ALYAO=Alya.x.orig ALYAM=Alya.x.modif ALYAPATH=/home/lschnorr/Trunk.6515.Alya for NP in 128; do for INFINIBAND in true false; do for CASE in metis sfc; do for ALYA in $ALYAO $ALYAM; do NPN=$((NP)) UNIQUE=${EID}_${CLUSTER}_${NODES}_${NPN}_${CASE}_${INFINIBAND}_${ALYA} echo "# " echo "# Experiment $UNIQUE" echo "# " echo "./scripts/run_alya_experiment.sh ${MACHINEFILE} ${NPN} ${ALYAPATH}/Executables/unix/$ALYA /home/lschnorr/WORK-RICARD/resp_${CASE}/fensap.dat $UNIQUE ${INFINIBAND}" echo "cp -prfv $UNIQUE.dir ${EDIR}" echo "rm -rf $UNIQUE.dir" done done done >> ${SCRIPT} done echo "# " >> ${SCRIPT} echo "# Final" >> ${SCRIPT} echo "# " >> ${SCRIPT} echo "cp $SCRIPT $EDIR" >> $SCRIPT chmod 755 ${SCRIPT} cat ${SCRIPT} echo "This script is at '${SCRIPT}', commit it." #+end_src #+RESULTS: #+begin_example if [ -e /home/lschnorr//exp_16-v1_grimoire_8 ]; then echo "Directory/File '/home/lschnorr//exp_16-v1_grimoire_8' already exists. Remove it or rename it." exit fi mkdir -p /home/lschnorr//exp_16-v1_grimoire_8 # # Experiment 16-v1_grimoire_8_128_metis_true_Alya.x.orig # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.orig /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 16-v1_grimoire_8_128_metis_true_Alya.x.orig true cp -prfv 16-v1_grimoire_8_128_metis_true_Alya.x.orig.dir /home/lschnorr//exp_16-v1_grimoire_8 rm -rf 16-v1_grimoire_8_128_metis_true_Alya.x.orig.dir # # Experiment 16-v1_grimoire_8_128_metis_true_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 16-v1_grimoire_8_128_metis_true_Alya.x.modif true cp -prfv 16-v1_grimoire_8_128_metis_true_Alya.x.modif.dir /home/lschnorr//exp_16-v1_grimoire_8 rm -rf 16-v1_grimoire_8_128_metis_true_Alya.x.modif.dir # # Experiment 16-v1_grimoire_8_128_sfc_true_Alya.x.orig # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.orig /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 16-v1_grimoire_8_128_sfc_true_Alya.x.orig true cp -prfv 16-v1_grimoire_8_128_sfc_true_Alya.x.orig.dir /home/lschnorr//exp_16-v1_grimoire_8 rm -rf 16-v1_grimoire_8_128_sfc_true_Alya.x.orig.dir # # Experiment 16-v1_grimoire_8_128_sfc_true_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 16-v1_grimoire_8_128_sfc_true_Alya.x.modif true cp -prfv 16-v1_grimoire_8_128_sfc_true_Alya.x.modif.dir /home/lschnorr//exp_16-v1_grimoire_8 rm -rf 16-v1_grimoire_8_128_sfc_true_Alya.x.modif.dir # # Experiment 16-v1_grimoire_8_128_metis_false_Alya.x.orig # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.orig /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 16-v1_grimoire_8_128_metis_false_Alya.x.orig false cp -prfv 16-v1_grimoire_8_128_metis_false_Alya.x.orig.dir /home/lschnorr//exp_16-v1_grimoire_8 rm -rf 16-v1_grimoire_8_128_metis_false_Alya.x.orig.dir # # Experiment 16-v1_grimoire_8_128_metis_false_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_metis/fensap.dat 16-v1_grimoire_8_128_metis_false_Alya.x.modif false cp -prfv 16-v1_grimoire_8_128_metis_false_Alya.x.modif.dir /home/lschnorr//exp_16-v1_grimoire_8 rm -rf 16-v1_grimoire_8_128_metis_false_Alya.x.modif.dir # # Experiment 16-v1_grimoire_8_128_sfc_false_Alya.x.orig # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.orig /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 16-v1_grimoire_8_128_sfc_false_Alya.x.orig false cp -prfv 16-v1_grimoire_8_128_sfc_false_Alya.x.orig.dir /home/lschnorr//exp_16-v1_grimoire_8 rm -rf 16-v1_grimoire_8_128_sfc_false_Alya.x.orig.dir # # Experiment 16-v1_grimoire_8_128_sfc_false_Alya.x.modif # ./scripts/run_alya_experiment.sh machine-file 128 /home/lschnorr/Trunk.6515.Alya/Executables/unix/Alya.x.modif /home/lschnorr/WORK-RICARD/resp_sfc/fensap.dat 16-v1_grimoire_8_128_sfc_false_Alya.x.modif false cp -prfv 16-v1_grimoire_8_128_sfc_false_Alya.x.modif.dir /home/lschnorr//exp_16-v1_grimoire_8 rm -rf 16-v1_grimoire_8_128_sfc_false_Alya.x.modif.dir # # Final # cp script_16-v1_grimoire_8_nodes.sh /home/lschnorr//exp_16-v1_grimoire_8 This script is at 'script_16-v1_grimoire_8_nodes.sh', commit it. #+end_example * Analysis ** 2-node grisou (17 and 32 ranks) :EXP1: *** Rough epoch-based execution time #+name: exp_1_grisou_2_epoch #+begin_src shell :results output org table EDIR=exp_1_grisou_2 for case in $(find $EDIR | grep -e "sfc.org" -e "metis.org"); do echo -n "$(basename $case) " cat $case | grep epoch | sed "s/^.*: //" | tr '\n' '-' | sed -e "s/-$//" -e "s/-/ - /" -e "s/$/\n/" | bc -l; done | sed -e "s/.org//" | sed -e "s/_/|/g" -e "s/-//g" #+end_src #+RESULTS: exp_1_grisou_2_epoch #+BEGIN_SRC org | grisou | 2 | 32 | metis | 951 | | grisou | 2 | 32 | sfc | 762 | | grisou | 2 | 17 | metis | 1417 | | grisou | 2 | 17 | sfc | 1225 | #+END_SRC #+header: :var df=exp_1_grisou_2_epoch #+begin_src R :results output graphics :file img/exp_1_grisou_2_epoch.png :exports both :width 600 :height 400 :session library(dplyr); library(ggplot2); df %>% rename(Cluster= V1, Nodes = V2, NP = V3, Case = V4, Time = V5) %>% filter(NP != 121) %>% ggplot(aes(x=NP, y=Time, color=Case)) + theme_bw() + ylim(0,NA) + ylab("Time [s]") + geom_point() + geom_line() + theme(legend.position = "top"); #+end_src #+RESULTS: [[file:img/exp_1_grisou_2_epoch.png]] ** 10-node grisou (97, 113, 129, 145, 160) :EXP2: *** Rough epoch-based execution time #+name: exp_2_grisou_10_epoch #+begin_src shell :results output org table EDIR=exp_2_grisou_10 for case in $(find $EDIR | grep -e "sfc.org" -e "metis.org"); do echo -n "$(basename $case) " cat $case | grep epoch | sed "s/^.*: //" | tr '\n' '-' | sed -e "s/-$//" -e "s/-/ - /" -e "s/$/\n/" | bc -l; done | sed -e "s/.org//" | sed -e "s/_/|/g" -e "s/-//g" #+end_src #+RESULTS: exp_2_grisou_10_epoch #+BEGIN_SRC org | grisou | 10 | 160 | metis | 304 | | grisou | 10 | 129 | sfc | 337 | | grisou | 10 | 97 | metis | 401 | | grisou | 10 | 145 | sfc | 315 | | grisou | 10 | 129 | metis | 345 | | grisou | 10 | 145 | metis | 321 | | grisou | 10 | 97 | sfc | 380 | | grisou | 10 | 160 | sfc | 301 | | grisou | 10 | 113 | metis | 363 | | grisou | 10 | 113 | sfc | 359 | #+END_SRC #+header: :var df=exp_2_grisou_10_epoch #+begin_src R :results output graphics :file img/exp_2_grisou_10_epoch.png :exports both :width 200 :height 400 :session library(dplyr); library(ggplot2); df %>% rename(Cluster= V1, Nodes = V2, NP = V3, Case = V4, Time = V5) %>% filter(NP != 121) %>% ggplot(aes(x=NP, y=Time, color=Case)) + theme_bw() + ylim(0,NA) + ylab("Time [s]") + geom_point() + geom_line() + theme(legend.position = "top"); #+end_src #+RESULTS: [[file:img/exp_2_grisou_10_epoch.png]] *** Convert to CSV I need on the $PATH: - akypuera's otf22paje (compiled using OTF2 libraries of ScoreP 3.0) - pajeng's =pj_dump= #+begin_src shell :results output export PATH=$PATH:/home/schnorr/install/stow/bin/ export PATH=$PATH:/home/schnorr/dev/pajeng/b/ convert() { pushd $(dirname $otf2) otf22paje traces.otf2 | pj_dump | grep ^State | cut -d, -f2,4,5,8 | sed -e "s/ //g" -e "s/MPIRank//" > traces.csv popd } EDIR=exp_2_grisou_10 N=4 # number of parallel batches for otf2 in $(find $EDIR | grep otf2$); do ((i=i%N)); ((i++==0)) && wait convert $otf2 & done #+end_src #+RESULTS: #+begin_example /home/schnorr/install/stow/bin//otf22paje Usage: otf22paje [OPTION...] ANCHORFILE Converts an OTF2 archive to the Paje file format -b, --basic Avoid extended events (impoverished trace file) -d, --dummy Read input traces but won't translate (no output) -f, --flat Flat hierarchy (only MPI ranks) -h, --hostfile=FILE MPI hostfile to create system hierarchy -i, --ignore-errors Ignore errors -l, --no-links Don't convert links -m, --comment=COMMENT Comment is echoed to output -n, --commentfile=FILE Comments (from file) echoed to output -o, --only-mpi Only convert MPI states -s, --no-states Don't convert states -z, --normalize-mpi Try to normalize MPI state names -?, --help Give this help list --usage Give a short usage message Mandatory or optional arguments to long options are also mandatory or optional for any corresponding short options. ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_160_metis.dir/scorep_grisou_10_160_metis ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_129_sfc.dir/scorep_grisou_10_129_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_97_metis.dir/scorep_grisou_10_97_metis ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_145_sfc.dir/scorep_grisou_10_145_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_129_metis.dir/scorep_grisou_10_129_metis ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_145_metis.dir/scorep_grisou_10_145_metis ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_97_sfc.dir/scorep_grisou_10_97_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_160_sfc.dir/scorep_grisou_10_160_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_113_metis.dir/scorep_grisou_10_113_metis ~/dev/Alya-Perf ~/dev/Alya-Perf ~/dev/Alya-Perf/exp_2_grisou_10/grisou_10_113_sfc.dir/scorep_grisou_10_113_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf #+end_example #+begin_src shell :results output export PATH=$PATH:/home/schnorr/install/stow/bin/ export PATH=$PATH:/home/schnorr/dev/pajeng/b/ convert() { pushd $(dirname $otf2) otf22paje traces.otf2 | pj_dump | grep ^State | cut -d, -f2,4,5,8 | sed -e "s/ //g" -e "s/MPIRank//" > traces.csv popd } EDIR=exp_2_grisou_10 # Files already converted (whose CSV size is not zero) EXISTINGFILE=$(tempfile) OTF2FILE=$(tempfile) find $EDIR -not -empty | grep csv$ | sed -e "s/.csv$//" | sort > $EXISTINGFILE find $EDIR | grep otf2 | sed -e "s/.otf2$//" | sort > $OTF2FILE for otf2 in $(comm -3 $OTF2FILE $EXISTINGFILE | sed "s/$/.otf2/"); do convert $otf2 done #+end_src #+RESULTS: *** Read and analysis Check read and analysis of next section. ** 10-node grisou (97, 113, 129, 145, 160) :EXP3: *** Rough epoch-based execution time #+name: exp_3_grisou_10_epoch #+begin_src shell :results output org table EDIR=exp_3_grisou_10 for case in $(find $EDIR | grep -e "sfc.org" -e "metis.org"); do echo -n "$(basename $case) " cat $case | grep epoch | sed "s/^.*: //" | tr '\n' '-' | sed -e "s/-$//" -e "s/-/ - /" -e "s/$/\n/" | bc -l; done | sed -e "s/.org//" | sed -e "s/_/|/g" -e "s/-//g" #+end_src #+RESULTS: exp_3_grisou_10_epoch #+BEGIN_SRC org | 3 | grisou | 10 | 145 | sfc | 320 | | 3 | grisou | 10 | 113 | sfc | 356 | | 3 | grisou | 10 | 97 | metis | 394 | | 3 | grisou | 10 | 129 | sfc | 336 | | 3 | grisou | 10 | 129 | metis | 344 | | 3 | grisou | 10 | 160 | sfc | 308 | | 3 | grisou | 10 | 145 | metis | 319 | | 3 | grisou | 10 | 113 | metis | 362 | | 3 | grisou | 10 | 160 | metis | 306 | | 3 | grisou | 10 | 97 | sfc | 380 | #+END_SRC #+header: :var df=exp_3_grisou_10_epoch #+begin_src R :results output graphics :file img/exp_3_grisou_10_epoch.png :exports both :width 200 :height 400 :session library(dplyr); library(ggplot2); df %>% rename(EID = V1, Cluster= V2, Nodes = V3, NP = V4, Case = V5, Time = V6) %>% filter(NP != 121) %>% ggplot(aes(x=NP, y=Time, color=Case)) + theme_bw() + ylim(0,NA) + ylab("Time [s]") + geom_point() + geom_line() + theme(legend.position = "top"); #+end_src #+RESULTS: [[file:img/exp_3_grisou_10_epoch.png]] *** Convert to CSV I need on the $PATH: - akypuera's otf22paje (compiled using OTF2 libraries of ScoreP 3.0) - pajeng's =pj_dump= #+begin_src shell :results output export PATH=$PATH:/home/schnorr/install/stow/bin/ export PATH=$PATH:/home/schnorr/dev/pajeng/b/ convert() { pushd $(dirname $otf2) otf22paje traces.otf2 | pj_dump | grep ^State | cut -d, -f2,4,5,8 | sed -e "s/ //g" -e "s/MPIRank//" > traces.csv popd } EDIR=exp_3_grisou_10 # Files already converted (whose CSV size is not zero) EXISTINGFILE=$(tempfile) OTF2FILE=$(tempfile) find $EDIR -not -empty | grep csv$ | sed -e "s/.csv$//" | sort > $EXISTINGFILE find $EDIR | grep otf2 | sed -e "s/.otf2$//" | sort > $OTF2FILE for otf2 in $(comm -3 $OTF2FILE $EXISTINGFILE | sed "s/$/.otf2/"); do echo $otf2 convert $otf2 done #+end_src #+RESULTS: #+begin_example exp_3_grisou_10/3_grisou_10_113_metis.dir/scorep_3_grisou_10_113_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_113_metis.dir/scorep_3_grisou_10_113_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_113_sfc.dir/scorep_3_grisou_10_113_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_113_sfc.dir/scorep_3_grisou_10_113_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_129_metis.dir/scorep_3_grisou_10_129_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_129_metis.dir/scorep_3_grisou_10_129_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_129_sfc.dir/scorep_3_grisou_10_129_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_129_sfc.dir/scorep_3_grisou_10_129_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_145_metis.dir/scorep_3_grisou_10_145_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_145_metis.dir/scorep_3_grisou_10_145_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_145_sfc.dir/scorep_3_grisou_10_145_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_145_sfc.dir/scorep_3_grisou_10_145_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_160_metis.dir/scorep_3_grisou_10_160_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_160_metis.dir/scorep_3_grisou_10_160_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_160_sfc.dir/scorep_3_grisou_10_160_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_160_sfc.dir/scorep_3_grisou_10_160_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_97_metis.dir/scorep_3_grisou_10_97_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_97_metis.dir/scorep_3_grisou_10_97_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_97_sfc.dir/scorep_3_grisou_10_97_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_97_sfc.dir/scorep_3_grisou_10_97_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf #+end_example *** Read in R #+begin_src R :results output :session :exports both :tangle do.R :tangle-mode (identity #o755) #!/usr/bin/Rscript library(readr); library(dplyr); alya_scorep_trace_read <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); read_csv(filename, col_names=c("Rank", "Start", "End", "Value"), progress=TRUE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, grepl("endste", .$Value) ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum- mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = cumsum(Iteration)) %>% mutate(EID = meta[2], Platform = meta[3], Nodes = meta[4], NP = meta[5], Partitioning = meta[6]); } alya_scorep_trace_iterations <- function(filename) { alya_scorep_trace_read(filename) %>% group_by(Rank, Iteration, Platform, Nodes, NP, Partitioning, EID) %>% summarize(N=n(), S=min(Start), E=max(End)); } args = commandArgs(trailingOnly=TRUE) print(args); df <- do.call("rbind", lapply(args, function(x) { alya_scorep_trace_iterations(x) })); write.csv(df, "exp2_exp3_iterations.csv"); #+end_src #+RESULTS: *** Analysis :ATTACH: :PROPERTIES: :Attachments: exp2_exp3_iterations.csv :ID: d2233039-ad84-42ac-ad70-545877ba7e8e :END: **** Iteration-only shows SFC is worse than Metis (average per-iteration span) #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/d2/233039-ad84-42ac-ad70-545877ba7e8e/exp2_exp3_iterations.csv") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Attaching package: ‘dplyr’ The following objects are masked from ‘package:stats’: filter, lag The following objects are masked from ‘package:base’: intersect, setdiff, setequal, union Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_integer(), N = col_integer(), S = col_double(), E = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grisou:25760 10:25760 Min. : 97.0 1st Qu.: 32.00 1st Qu.: 3.0 1st Qu.:113.0 Median : 64.00 Median : 5.5 Median :129.0 Mean : 65.84 Mean : 5.5 Mean :132.7 3rd Qu.: 96.00 3rd Qu.: 8.0 3rd Qu.:145.0 Max. :159.00 Max. :10.0 Max. :160.0 Partitioning EID N Start End metis:12880 Min. :2.0 Min. : 3580 Min. :120.2 Min. :139.9 sfc :12880 1st Qu.:2.0 1st Qu.:17618 1st Qu.:168.8 1st Qu.:188.9 Median :2.5 Median :24816 Median :220.5 Median :241.4 Mean :2.5 Mean :28263 Mean :222.6 Mean :243.2 3rd Qu.:3.0 3rd Qu.:37184 3rd Qu.:271.5 3rd Qu.:293.7 Max. :3.0 Max. :85082 Max. :375.5 Max. :397.2 Duration Min. :13.57 1st Qu.:18.05 Median :19.93 Mean :20.63 3rd Qu.:23.17 Max. :29.10 #+end_example Execution time #+begin_src R :results output graphics :file img/exp2_exp3_iterations_only.png :exports both :width 600 :height 400 :session df %>% group_by(Platform, Nodes, NP, Partitioning, EID) %>% summarize(Time = max(End) - min(Start)) %>% group_by(Platform, Nodes, NP, Partitioning) %>% summarize(Time = mean(Time), N=n()) %>% print %>% ggplot(aes(x=NP, y=Time, color=Partitioning)) + theme_bw(base_size=12) + geom_point() + geom_line() + ylim(0, NA) #+end_src #+RESULTS: [[file:img/exp2_exp3_iterations_only.png]] #+begin_src R :results output :session :exports both df %>% group_by(NP, Iteration, Partitioning) %>% # calculate the mean of number of operations per iteration summarize(N=mean(N), Z=n()); #+end_src #+RESULTS: #+begin_example Source: local data frame [100 x 5] Groups: NP, Iteration [?] NP Iteration Partitioning N Z 1 97 1 metis 19204.61 194 2 97 1 sfc 40975.99 194 3 97 2 metis 18361.41 194 4 97 2 sfc 39909.78 194 5 97 3 metis 18358.87 194 6 97 3 sfc 39712.10 194 7 97 4 metis 17679.86 194 8 97 4 sfc 38238.62 194 9 97 5 metis 17201.62 194 10 97 5 sfc 37063.47 194 # ... with 90 more rows #+end_example #+begin_src R :results output graphics :file img/exp2_exp3_number_of_operations.png :exports both :width 600 :height 400 :session library(ggplot2); df %>% group_by(NP, Iteration, Partitioning) %>% # calculate the mean of number of operations per iteration summarize(N=mean(N), Z=n()) %>% ungroup() %>% ggplot(aes(x=Iteration, y=N, color=Partitioning)) + theme_bw(base_size=12) + ylim(0,NA) + geom_point() + theme(legend.position="top") + facet_wrap(~NP, nrow=1) #+end_src #+RESULTS: [[file:img/exp2_exp3_number_of_operations.png]] **** Addressing Rick's questions (on max per-iteration) :RICK: Could you give us the results shown in =exp2_exp3_iterations_only.png=, but considering the maximum time for each CPU-core instead of the average? :END: :LUCAS: If I understood correctly your question, you want to see the time each CPU-core took to compute the 10 iterations. So, I put Rank in the X axis, and on Y, I put the difference of time between the moment where a given rank enters the first iteration until the moment it leaves the 10th iteration. I select only one experiment (exp2) because they are very similar (on the same set of machines). The exp2 experiment contained 5 executions with different NP (97, 113, 129, 145, and 160). The five plots are in the attached file =exp2_exp3_10iter_time_of_each_rank.png= :END: #+begin_src R :results output graphics :file img/exp2_10iter_time_of_each_rank.png :exports both :width 600 :height 600 :session library(ggplot2); df %>% filter(EID==2) %>% group_by(Rank, NP, Partitioning, EID) %>% summarize(Time = max(End) - min(Start)) %>% ggplot(aes(x=Rank, y=Time, color=Partitioning)) + theme_bw(base_size=12) + geom_point() + ylim(0, NA) + facet_grid(EID~NP) + theme(legend.position="top") #+end_src #+RESULTS: [[file:img/exp2_10iter_time_of_each_rank.png]] :LUCAS: Since I am not sure I understood correctly your request, the plot =img/exp2_exp3_iterations_only_max.png= contains the max execution time (on Y) of each of the 10 iterations (the horizontal facets) taking into account all ranks belonging to each of the five NP experiments (on X). :END: #+begin_src R :results output graphics :file img/exp2_exp3_iterations_only_max.png :exports both :width 1200 :height 400 :session library(ggplot2); df %>% select(-Start, -End) %>% group_by(Iteration, NP, Partitioning) %>% summarize(Time = max(Duration), N=n()) %>% ungroup() %>% mutate(NP=as.factor(NP)) %>% ggplot(aes(x=NP, y=Time, color=Partitioning, group=Partitioning)) + theme_bw(base_size=12) + geom_point() + geom_line() + ylim(0, NA) + facet_grid(~Iteration) + theme(legend.position="top") #+end_src #+RESULTS: [[file:img/exp2_exp3_iterations_only_max.png]] ** 23-node nova (367 352 336 272 208 144 80 16) :EXP4: *** Preparation (2-nodes only) **** Convert to CSV I need on the $PATH: - akypuera's otf22paje (compiled using OTF2 libraries of ScoreP 3.0) - pajeng's =pj_dump= #+begin_src shell :results output export PATH=$PATH:/home/schnorr/install/stow/bin/ export PATH=$PATH:/home/schnorr/dev/pajeng/b/ convert() { pushd $(dirname $otf2) otf22paje traces.otf2 | pj_dump | grep ^State | cut -d, -f2,4,5,8 | sed -e "s/ //g" -e "s/MPIRank//" | gzip > traces.csv.gz popd } EDIR=exp_prep-4-v2_grisou_4 # Files already converted (whose CSV size is not zero) EXISTINGFILE=$(tempfile) OTF2FILE=$(tempfile) find $EDIR -not -empty | grep csv$ | sed -e "s/.csv$//" | sort > $EXISTINGFILE find $EDIR | grep otf2 | sed -e "s/.otf2$//" | sort > $OTF2FILE for otf2 in $(comm -3 $OTF2FILE $EXISTINGFILE | sed "s/$/.otf2/"); do echo $otf2 convert $otf2 done #+end_src #+RESULTS: : exp_prep-4-v2_grisou_4/prep-4_grisou_4_63_metis.dir/scorep_prep-4_grisou_4_63_metis/traces.otf2 : ~/dev/Alya-Perf/exp_prep-4-v2_grisou_4/prep-4_grisou_4_63_metis.dir/scorep_prep-4_grisou_4_63_metis ~/dev/Alya-Perf : ~/dev/Alya-Perf : exp_prep-4-v2_grisou_4/prep-4_grisou_4_63_sfc.dir/scorep_prep-4_grisou_4_63_sfc/traces.otf2 : ~/dev/Alya-Perf/exp_prep-4-v2_grisou_4/prep-4_grisou_4_63_sfc.dir/scorep_prep-4_grisou_4_63_sfc ~/dev/Alya-Perf : ~/dev/Alya-Perf **** Extract iteration information :ATTACH: :PROPERTIES: :Attachments: exp_prep-4_iterations.csv.gz :ID: b0e3978c-4131-4005-b979-a86aad031816 :END: #+begin_src shell :results output ./do.R \ exp_prep-4-v1_grisou_4/prep-4_grisou_4_18_sfc.dir/scorep_prep-4_grisou_4_18_sfc/traces.csv.gz \ exp_prep-4-v1_grisou_4/prep-4_grisou_4_33_sfc.dir/scorep_prep-4_grisou_4_33_sfc/traces.csv.gz \ exp_prep-4-v1_grisou_4/prep-4_grisou_4_49_metis.dir/scorep_prep-4_grisou_4_49_metis/traces.csv.gz \ exp_prep-4-v1_grisou_4/prep-4_grisou_4_63_sfc.dir/scorep_prep-4_grisou_4_63_sfc/traces.csv.gz \ exp_prep-4-v1_grisou_4/prep-4_grisou_4_33_metis.dir/scorep_prep-4_grisou_4_33_metis/traces.csv.gz \ exp_prep-4-v1_grisou_4/prep-4_grisou_4_18_metis.dir/scorep_prep-4_grisou_4_18_metis/traces.csv.gz \ exp_prep-4-v1_grisou_4/prep-4_grisou_4_49_sfc.dir/scorep_prep-4_grisou_4_49_sfc/traces.csv.gz \ exp_prep-4-v1_grisou_4/prep-4_grisou_4_63_metis.dir/scorep_prep-4_grisou_4_63_metis/traces.csv.gz \ exp_prep-4-v2_grisou_4/prep-4_grisou_4_63_sfc.dir/scorep_prep-4_grisou_4_63_sfc/traces.csv.gz \ exp_prep-4-v2_grisou_4/prep-4_grisou_4_63_metis.dir/scorep_prep-4_grisou_4_63_metis/traces.csv.gz \ #+end_src Results are saved in file compressed CSV file called =exp_prep-4.csv.gz=. **** Analysis Forthcoming. *** 23-Node data **** Convert to CSV I need on the $PATH: - akypuera's otf22paje (compiled using OTF2 libraries of ScoreP 3.0) - pajeng's =pj_dump= #+begin_src shell :results output export PATH=$PATH:/home/lschnorr/akypuera/b/ export PATH=$PATH:/home/lschnorr/pajeng/b/ convert() { pushd $(dirname $otf2) otf22paje traces.otf2 | pj_dump | grep ^State | cut -d, -f2,4,5,8 | sed -e "s/ //g" -e "s/MPIRank//" | gzip > traces.csv.gz popd } EDIR=exp_4_grisou_23_v1/ # Files already converted (whose CSV size is not zero) EXISTINGFILE=$(tempfile) OTF2FILE=$(tempfile) find $EDIR -not -empty | grep csv.gz$ | sed -e "s/.csv.gz$//" | sort > $EXISTINGFILE find $EDIR | grep \.otf2$ | sed -e "s/.otf2$//" | sort > $OTF2FILE for otf2 in $(comm -3 $OTF2FILE $EXISTINGFILE | sed "s/$/.otf2/"); do echo $otf2 convert $otf2 done #+end_src #+RESULTS: #+begin_example exp_3_grisou_10/3_grisou_10_113_metis.dir/scorep_3_grisou_10_113_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_113_metis.dir/scorep_3_grisou_10_113_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_113_sfc.dir/scorep_3_grisou_10_113_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_113_sfc.dir/scorep_3_grisou_10_113_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_129_metis.dir/scorep_3_grisou_10_129_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_129_metis.dir/scorep_3_grisou_10_129_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_129_sfc.dir/scorep_3_grisou_10_129_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_129_sfc.dir/scorep_3_grisou_10_129_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_145_metis.dir/scorep_3_grisou_10_145_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_145_metis.dir/scorep_3_grisou_10_145_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_145_sfc.dir/scorep_3_grisou_10_145_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_145_sfc.dir/scorep_3_grisou_10_145_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_160_metis.dir/scorep_3_grisou_10_160_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_160_metis.dir/scorep_3_grisou_10_160_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_160_sfc.dir/scorep_3_grisou_10_160_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_160_sfc.dir/scorep_3_grisou_10_160_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_97_metis.dir/scorep_3_grisou_10_97_metis/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_97_metis.dir/scorep_3_grisou_10_97_metis ~/dev/Alya-Perf ~/dev/Alya-Perf exp_3_grisou_10/3_grisou_10_97_sfc.dir/scorep_3_grisou_10_97_sfc/traces.otf2 ~/dev/Alya-Perf/exp_3_grisou_10/3_grisou_10_97_sfc.dir/scorep_3_grisou_10_97_sfc ~/dev/Alya-Perf ~/dev/Alya-Perf #+end_example **** Post-processing in R #+begin_src R :results output :session :exports both :tangle do-exp4.R :tangle-mode (identity #o755) #!/usr/bin/Rscript library(readr); library(dplyr); alya_scorep_trace_read <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); read_csv(filename, col_names=c("Rank", "Start", "End", "Value"), progress=TRUE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, grepl("endste", .$Value) ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = as.integer(cumsum(Iteration))) %>% ungroup() %>% # Define metadata mutate(EID = meta[2], Platform = meta[3], Nodes = meta[4], NP = meta[5], Partitioning = meta[6]); } alya_scorep_trace_iterations <- function(filename) { alya_scorep_trace_read(filename) %>% group_by(Rank, Iteration, Platform, Nodes, NP, Partitioning, EID) %>% filter(grepl("MPI_", Value)) %>% summarize(N=n(), S=min(Start), E=max(End), Comm=sum(End-Start), Comp=(E-S)-Comm); } args = commandArgs(trailingOnly=TRUE) print(args); df <- do.call("rbind", lapply(args, function(x) { alya_scorep_trace_iterations(x) })); write.csv(df, "exp4_iterations.csv"); #+end_src **** Read in R :ATTACH: :PROPERTIES: :Attachments: exp4_iterations.csv.gz :ID: 3b264f0c-58ba-47c0-b291-b5729da5e669 :END: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/3b/264f0c-58ba-47c0-b291-b5729da5e669/exp4_iterations.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_integer(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.0 Min. : 1.000 grisou:19613 23:19613 Min. : 17.0 1st Qu.: 61.0 1st Qu.: 2.000 1st Qu.:209.0 Median :132.0 Median : 5.000 Median :337.0 Mean :145.2 Mean : 5.091 Mean :291.5 3rd Qu.:221.0 3rd Qu.: 8.000 3rd Qu.:353.0 Max. :367.0 Max. :10.000 Max. :368.0 Partitioning EID N Start End metis: 1783 Min. :4 Min. : 3579 Min. : 131.6 Min. : 145.8 sfc :17830 1st Qu.:4 1st Qu.: 29737 1st Qu.: 151.6 1st Qu.: 166.4 Median :4 Median : 38587 Median : 189.7 Median : 201.3 Mean :4 Mean : 39148 Mean : 199.2 Mean : 214.1 3rd Qu.:4 3rd Qu.: 47123 3rd Qu.: 223.9 3rd Qu.: 234.6 Max. :4 Max. :103825 Max. :1417.6 Max. :1527.0 Comm Comp Duration Min. : 0.539 Min. : 0.01607 Min. : 8.054 1st Qu.: 5.304 1st Qu.: 4.92897 1st Qu.: 10.519 Median : 5.926 Median : 5.75274 Median : 11.189 Mean : 6.596 Mean : 8.34015 Mean : 14.936 3rd Qu.: 7.522 3rd Qu.: 8.05452 3rd Qu.: 15.129 Max. :169.634 Max. :167.55322 Max. :169.672 #+end_example **** Plot #+begin_src R :results output :session :exports both df %>% filter(Partitioning == "metis") %>% tail #+end_src #+RESULTS: #+begin_example # A tibble: 6 x 13 Rank Iteration Platform Nodes NP Partitioning EID N Start 1 331 1 grisou 23 337 metis 4 19477 149.0451 2 332 1 grisou 23 337 metis 4 31973 149.0405 3 333 1 grisou 23 337 metis 4 35097 149.0563 4 334 1 grisou 23 337 metis 4 35097 149.0551 5 335 1 grisou 23 337 metis 4 41345 149.0725 6 336 1 grisou 23 337 metis 4 28849 149.0405 # ... with 4 more variables: End , Comm , Comp , Duration #+end_example #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 500 :height 400 :session library(ggplot2); library(tidyr); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID) %>% group_by (Iteration, Platform, Nodes, NP, Partitioning, EID, Variable) %>% summarize(Mean = mean(Value), SE = 3*sd(Value)/sqrt(n()), N=n()) %>% filter(NP == 368) %>% ggplot(aes(x=Iteration, y=Mean, ymin=Mean-SE, ymax=Mean+SE, color=Partitioning)) + theme_bw(base_size=12) + ylim(0,NA) + geom_point() + geom_errorbar(width=.3) + geom_line () + theme(legend.position="top") + facet_grid(NP~Variable) #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure24763z4R.png]] Unfortunately, Metis as misconfigured in lyon, it ran only 1 iteration. **** Plot with min/max #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 500 :height 400 :session library(ggplot2); library(tidyr); df %>% filter(Rank != 0) %>% filter(Partitioning == "sfc") %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID) %>% filter(NP == 368) %>% ggplot(aes(x=Iteration, y=Value, group=Iteration)) + theme_bw(base_size=12) + ylim(0,NA) + geom_boxplot() + theme(legend.position="top") + facet_grid(Partitioning~Variable) #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure24763AKM.png]] **** Gantt to check MPI calls ***** Read the trace #+begin_src R :results output :session :exports both library(readr); library(dplyr); alya_scorep_trace_read <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); read_csv(filename, col_names=c("Rank", "Start", "End", "Value"), progress=TRUE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, grepl("endste", .$Value) ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = as.integer(cumsum(Iteration))) %>% ungroup() %>% # Define metadata mutate(EID = meta[2], Platform = meta[3], Nodes = meta[4], NP = meta[5], Partitioning = meta[6]); } TRACE <- "exp_prep-4-v1_grisou_4/prep-4_grisou_4_63_sfc.dir/scorep_prep-4_grisou_4_63_sfc/traces.csv.gz" df4.mpi <- alya_scorep_trace_read(TRACE); #+end_src #+RESULTS: : Parsed with column specification: : cols( : Rank = col_integer(), : Start = col_double(), : End = col_double(), : Value = col_character() : ) : |= | 2% 22 MB |= | 2% 22 MB |= | 2% 22 MB |= | 2% 22 MB |= | 2% 22 MB |= | 2% 22 MB |= | 2% 22 MB |= | 2% 23 MB |= | 2% 23 MB |= | 3% 23 MB |= | 3% 23 MB |= | 3% 23 MB |= | 3% 23 MB |= | 3% 23 MB |= | 3% 23 MB |= | 3% 23 MB |== | 3% 23 MB |== | 3% 23 MB |== | 3% 23 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 24 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 25 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 26 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 27 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 28 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 29 MB |== | 3% 30 MB |== | 3% 30 MB |== | 3% 30 MB |== | 3% 30 MB |== | 3% 30 MB |== | 3% 30 MB |== | 3% 30 MB |== | 3% 30 MB |== | 3% 30 MB |== | 3% 30 MB |== | 4% 30 MB |== | 4% 30 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 31 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 32 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 33 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 34 MB |== | 4% 35 MB |== | 4% 35 MB |== | 4% 35 MB |== | 4% 35 MB |== | 4% 35 MB |== | 4% 35 MB |== | 4% 35 MB |=== | 4% 35 MB |=== | 4% 35 MB |=== | 4% 35 MB |=== | 4% 35 MB |=== | 4% 35 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 36 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 37 MB |=== | 4% 38 MB |=== | 4% 38 MB |=== | 4% 38 MB |=== | 4% 38 MB |=== | 4% 38 MB |=== | 4% 38 MB |=== | 4% 38 MB |=== | 5% 38 MB |=== | 5% 38 MB |=== | 5% 38 MB |=== | 5% 38 MB |=== | 5% 38 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 39 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 40 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 41 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 42 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 43 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 44 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 45 MB |=== | 5% 46 MB |=== | 5% 46 MB |=== | 5% 46 MB |=== | 6% 46 MB |=== | 6% 46 MB |=== | 6% 46 MB |=== | 6% 46 MB |=== | 6% 46 MB |=== | 6% 46 MB |=== | 6% 46 MB |=== | 6% 46 MB |=== | 6% 46 MB |=== | 6% 47 MB |=== | 6% 47 MB |=== | 6% 47 MB |=== | 6% 47 MB |=== | 6% 47 MB |==== | 6% 47 MB |==== | 6% 47 MB |==== | 6% 47 MB |==== | 6% 47 MB |==== | 6% 47 MB |==== | 6% 47 MB |==== | 6% 47 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 48 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 49 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 50 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 51 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 52 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 6% 53 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 54 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 55 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 56 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 57 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 58 MB |==== | 7% 59 MB |==== | 7% 59 MB |==== | 7% 59 MB |==== | 7% 59 MB |===== | 7% 59 MB |===== | 7% 59 MB |===== | 7% 59 MB |===== | 7% 59 MB |===== | 7% 59 MB |===== | 7% 59 MB |===== | 7% 59 MB |===== | 7% 59 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 60 MB |===== | 7% 61 MB |===== | 7% 61 MB |===== | 7% 61 MB |===== | 7% 61 MB |===== | 7% 61 MB |===== | 7% 61 MB |===== | 7% 61 MB |===== | 7% 61 MB |===== | 8% 61 MB |===== | 8% 61 MB |===== | 8% 61 MB |===== | 8% 61 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 62 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 63 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 64 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 65 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 66 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 67 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 68 MB |===== | 8% 69 MB |===== | 8% 69 MB |===== | 8% 69 MB |===== | 8% 69 MB |===== | 8% 69 MB |===== | 9% 69 MB |===== | 9% 69 MB |===== | 9% 69 MB |===== | 9% 69 MB |===== | 9% 69 MB |===== | 9% 69 MB |===== | 9% 69 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 70 MB |===== | 9% 71 MB |===== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 71 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 72 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 73 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 74 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 75 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 76 MB |====== | 9% 77 MB |====== | 9% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 77 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 78 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 79 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 80 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 81 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 82 MB |====== | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 83 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 10% 84 MB |======= | 11% 84 MB |======= | 11% 84 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 85 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 86 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 87 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 88 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 89 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 90 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 91 MB |======= | 11% 92 MB |======= | 11% 92 MB |======= | 11% 92 MB |======= | 11% 92 MB |======= | 11% 92 MB |======= | 11% 92 MB |======= | 12% 92 MB |======= | 12% 92 MB |======= | 12% 92 MB |======= | 12% 92 MB |======= | 12% 92 MB |======= | 12% 92 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 93 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======= | 12% 94 MB |======== | 12% 94 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 95 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 96 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 97 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 98 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 99 MB |======== | 12% 100 MB |======== | 12% 100 MB |======== | 12% 100 MB |======== | 13% 100 MB |======== | 13% 100 MB |======== | 13% 100 MB |======== | 13% 100 MB |======== | 13% 100 MB |======== | 13% 100 MB |======== | 13% 100 MB |======== | 13% 100 MB |======== | 13% 100 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 101 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 102 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 103 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 104 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 105 MB |======== | 13% 106 MB |======== | 13% 106 MB |======== | 13% 106 MB |======== | 13% 106 MB |======== | 13% 106 MB |======== | 13% 106 MB |======== | 13% 106 MB |======== | 13% 106 MB |======== | 13% 106 MB |========= | 13% 106 MB |========= | 13% 106 MB |========= | 13% 106 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 13% 107 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 108 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 109 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 110 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 111 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 112 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 113 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 114 MB |========= | 14% 115 MB |========= | 14% 115 MB |========= | 14% 115 MB |========= | 14% 115 MB |========= | 14% 115 MB |========= | 14% 115 MB |========= | 14% 115 MB |========= | 14% 115 MB |========= | 15% 115 MB |========= | 15% 115 MB |========= | 15% 115 MB |========= | 15% 115 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 116 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 117 MB |========= | 15% 118 MB |========= | 15% 118 MB |========= | 15% 118 MB |========= | 15% 118 MB |========= | 15% 118 MB |========= | 15% 118 MB |========= | 15% 118 MB |========= | 15% 118 MB |========== | 15% 118 MB |========== | 15% 118 MB |========== | 15% 118 MB |========== | 15% 118 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 119 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 120 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 121 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 122 MB |========== | 15% 123 MB |========== | 15% 123 MB |========== | 15% 123 MB |========== | 15% 123 MB |========== | 15% 123 MB |========== | 16% 123 MB |========== | 16% 123 MB |========== | 16% 123 MB |========== | 16% 123 MB |========== | 16% 123 MB |========== | 16% 123 MB |========== | 16% 123 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 124 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 125 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 126 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 127 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 128 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 129 MB |========== | 16% 130 MB |========== | 16% 130 MB |========== | 16% 130 MB |========== | 16% 130 MB |========== | 16% 130 MB |========== | 16% 130 MB |=========== | 16% 130 MB |=========== | 16% 130 MB |=========== | 16% 130 MB |=========== | 16% 130 MB |=========== | 16% 130 MB |=========== | 16% 131 MB |=========== | 16% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 131 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 132 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 133 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 134 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 135 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 136 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 137 MB |=========== | 17% 138 MB |=========== | 17% 138 MB |=========== | 17% 138 MB |=========== | 17% 138 MB |=========== | 17% 138 MB |=========== | 17% 138 MB |=========== | 17% 138 MB |=========== | 17% 138 MB |=========== | 17% 138 MB |=========== | 18% 138 MB |=========== | 18% 138 MB |=========== | 18% 138 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 139 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 140 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 141 MB |=========== | 18% 142 MB |=========== | 18% 142 MB |=========== | 18% 142 MB |=========== | 18% 142 MB |============ | 18% 142 MB |============ | 18% 142 MB |============ | 18% 142 MB |============ | 18% 142 MB |============ | 18% 142 MB |============ | 18% 142 MB |============ | 18% 142 MB |============ | 18% 142 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 143 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 144 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 145 MB |============ | 18% 146 MB |============ | 18% 146 MB |============ | 18% 146 MB |============ | 18% 146 MB |============ | 18% 146 MB |============ | 18% 146 MB |============ | 19% 146 MB |============ | 19% 146 MB |============ | 19% 146 MB |============ | 19% 146 MB |============ | 19% 146 MB |============ | 19% 146 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 147 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 148 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 149 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 150 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 151 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 152 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 153 MB |============ | 19% 154 MB |============ | 19% 154 MB |============ | 19% 154 MB |============= | 20% 154 MB |============= | 20% 154 MB |============= | 20% 154 MB |============= | 20% 154 MB |============= | 20% 154 MB |============= | 20% 154 MB |============= | 20% 154 MB |============= | 20% 154 MB |============= | 20% 154 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 155 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 156 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 157 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 158 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 159 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 160 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 20% 161 MB |============= | 21% 161 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 162 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 163 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 164 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 165 MB |============= | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 166 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 167 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 168 MB |============== | 21% 169 MB |============== | 21% 169 MB |============== | 21% 169 MB |============== | 21% 169 MB |============== | 21% 169 MB |============== | 21% 169 MB |============== | 21% 169 MB |============== | 22% 169 MB |============== | 22% 169 MB |============== | 22% 169 MB |============== | 22% 169 MB |============== | 22% 169 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 170 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 171 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 172 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 173 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 174 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 175 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 176 MB |============== | 22% 177 MB |============== | 22% 177 MB |============== | 22% 177 MB |============== | 22% 177 MB |============== | 22% 177 MB |============== | 23% 177 MB |============== | 23% 177 MB |============== | 23% 177 MB |============== | 23% 177 MB |============== | 23% 177 MB |============== | 23% 177 MB |============== | 23% 177 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 178 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 179 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 180 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 181 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 182 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 183 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 184 MB |=============== | 23% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 185 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 186 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 187 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 188 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |=============== | 24% 189 MB |================ | 24% 189 MB |================ | 24% 189 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 190 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 191 MB |================ | 24% 192 MB |================ | 24% 192 MB |================ | 24% 192 MB |================ | 24% 192 MB |================ | 24% 192 MB |================ | 24% 192 MB |================ | 24% 192 MB |================ | 24% 192 MB |================ | 24% 192 MB |================ | 25% 192 MB |================ | 25% 192 MB |================ | 25% 192 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 193 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 194 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 195 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 196 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 197 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 198 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 199 MB |================ | 25% 200 MB |================ | 25% 200 MB |================ | 25% 200 MB |================ | 25% 200 MB |================ | 25% 200 MB |================ | 25% 200 MB |================ | 26% 200 MB |================ | 26% 200 MB |================ | 26% 200 MB |================ | 26% 200 MB |================ | 26% 200 MB |================ | 26% 200 MB |================ | 26% 201 MB |================ | 26% 201 MB |================ | 26% 201 MB |================ | 26% 201 MB |================ | 26% 201 MB |================ | 26% 201 MB |================ | 26% 201 MB |================ | 26% 201 MB |================= | 26% 201 MB |================= | 26% 201 MB |================= | 26% 201 MB |================= | 26% 201 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 202 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 203 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 204 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 205 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 206 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 207 MB |================= | 26% 208 MB |================= | 26% 208 MB |================= | 26% 208 MB |================= | 27% 208 MB |================= | 27% 208 MB |================= | 27% 208 MB |================= | 27% 208 MB |================= | 27% 208 MB |================= | 27% 208 MB |================= | 27% 208 MB |================= | 27% 208 MB |================= | 27% 208 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 209 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 210 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 211 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 212 MB |================= | 27% 213 MB |================= | 27% 213 MB |================= | 27% 213 MB |================= | 27% 213 MB |================= | 27% 213 MB |================= | 27% 213 MB |================= | 27% 213 MB |================== | 27% 213 MB |================== | 27% 213 MB |================== | 27% 213 MB |================== | 27% 213 MB |================== | 27% 213 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 214 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 27% 215 MB |================== | 28% 215 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 216 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 217 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 218 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 219 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 220 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 221 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 222 MB |================== | 28% 223 MB |================== | 28% 223 MB |================== | 28% 223 MB |================== | 28% 223 MB |================== | 28% 223 MB |================== | 28% 223 MB |================== | 28% 223 MB |================== | 29% 223 MB |================== | 29% 223 MB |================== | 29% 223 MB |================== | 29% 223 MB |================== | 29% 223 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 224 MB |================== | 29% 225 MB |================== | 29% 225 MB |================== | 29% 225 MB |================== | 29% 225 MB |=================== | 29% 225 MB |=================== | 29% 225 MB |=================== | 29% 225 MB |=================== | 29% 225 MB |=================== | 29% 225 MB |=================== | 29% 225 MB |=================== | 29% 225 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 226 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 227 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 228 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 229 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 230 MB |=================== | 29% 231 MB |=================== | 29% 231 MB |=================== | 29% 231 MB |=================== | 29% 231 MB |=================== | 30% 231 MB |=================== | 30% 231 MB |=================== | 30% 231 MB |=================== | 30% 231 MB |=================== | 30% 231 MB |=================== | 30% 231 MB |=================== | 30% 231 MB |=================== | 30% 231 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 232 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 233 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 234 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 235 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 236 MB |=================== | 30% 237 MB |=================== | 30% 237 MB |=================== | 30% 237 MB |==================== | 30% 237 MB |==================== | 30% 237 MB |==================== | 30% 237 MB |==================== | 30% 237 MB |==================== | 30% 237 MB |==================== | 30% 237 MB |==================== | 30% 237 MB |==================== | 30% 237 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 238 MB |==================== | 30% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 239 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 240 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 241 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 242 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 243 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 244 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 245 MB |==================== | 31% 246 MB |==================== | 31% 246 MB |==================== | 31% 246 MB |==================== | 31% 246 MB |==================== | 31% 246 MB |==================== | 31% 246 MB |==================== | 31% 246 MB |==================== | 31% 246 MB |==================== | 31% 246 MB |==================== | 32% 246 MB |==================== | 32% 246 MB |==================== | 32% 246 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 247 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 248 MB |==================== | 32% 249 MB |==================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 249 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 250 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 251 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 252 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 253 MB |===================== | 32% 254 MB |===================== | 32% 254 MB |===================== | 32% 254 MB |===================== | 32% 254 MB |===================== | 32% 254 MB |===================== | 32% 254 MB |===================== | 33% 254 MB |===================== | 33% 254 MB |===================== | 33% 254 MB |===================== | 33% 254 MB |===================== | 33% 254 MB |===================== | 33% 254 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 255 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 256 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 257 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 258 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 259 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |===================== | 33% 260 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 261 MB |====================== | 33% 262 MB |====================== | 33% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 262 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 263 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 264 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 265 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 266 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 267 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 268 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 34% 269 MB |====================== | 35% 269 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 270 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 271 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |====================== | 35% 272 MB |======================= | 35% 272 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 273 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 274 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 275 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 276 MB |======================= | 35% 277 MB |======================= | 35% 277 MB |======================= | 35% 277 MB |======================= | 35% 277 MB |======================= | 35% 277 MB |======================= | 35% 277 MB |======================= | 35% 277 MB |======================= | 36% 277 MB |======================= | 36% 277 MB |======================= | 36% 277 MB |======================= | 36% 277 MB |======================= | 36% 277 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 278 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 279 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 280 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 281 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 282 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 283 MB |======================= | 36% 284 MB |======================= | 36% 284 MB |======================= | 36% 284 MB |======================= | 36% 284 MB |======================= | 36% 284 MB |======================= | 36% 284 MB |======================= | 36% 284 MB |======================= | 36% 284 MB |======================= | 36% 284 MB |======================== | 36% 284 MB |======================== | 36% 284 MB |======================== | 36% 284 MB |======================== | 36% 285 MB |======================== | 36% 285 MB |======================== | 36% 285 MB |======================== | 36% 285 MB |======================== | 37% 285 MB |======================== | 37% 285 MB |======================== | 37% 285 MB |======================== | 37% 285 MB |======================== | 37% 285 MB |======================== | 37% 285 MB |======================== | 37% 285 MB |======================== | 37% 285 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 286 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 287 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 288 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 289 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 290 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 291 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 37% 292 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 293 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 294 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 295 MB |======================== | 38% 296 MB |======================== | 38% 296 MB |======================== | 38% 296 MB |======================== | 38% 296 MB |======================== | 38% 296 MB |======================== | 38% 296 MB |======================== | 38% 296 MB |========================= | 38% 296 MB |========================= | 38% 296 MB |========================= | 38% 296 MB |========================= | 38% 296 MB |========================= | 38% 296 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 297 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 298 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 299 MB |========================= | 38% 300 MB |========================= | 38% 300 MB |========================= | 38% 300 MB |========================= | 38% 300 MB |========================= | 38% 300 MB |========================= | 38% 300 MB |========================= | 38% 300 MB |========================= | 38% 300 MB |========================= | 38% 300 MB |========================= | 39% 300 MB |========================= | 39% 300 MB |========================= | 39% 300 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 301 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 302 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 303 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 304 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 305 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 306 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 307 MB |========================= | 39% 308 MB |========================= | 39% 308 MB |========================= | 39% 308 MB |========================= | 39% 308 MB |========================= | 39% 308 MB |========================== | 40% 308 MB |========================== | 40% 308 MB |========================== | 40% 308 MB |========================== | 40% 308 MB |========================== | 40% 308 MB |========================== | 40% 308 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 309 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 310 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 311 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 312 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 313 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 314 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 315 MB |========================== | 40% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 316 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 317 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 318 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 319 MB |========================== | 41% 320 MB |========================== | 41% 320 MB |========================== | 41% 320 MB |========================== | 41% 320 MB |=========================== | 41% 320 MB |=========================== | 41% 320 MB |=========================== | 41% 320 MB |=========================== | 41% 320 MB |=========================== | 41% 320 MB |=========================== | 41% 320 MB |=========================== | 41% 320 MB |=========================== | 41% 320 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 321 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 322 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 41% 323 MB |=========================== | 42% 323 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 324 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 325 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 326 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 327 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 328 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 329 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 330 MB |=========================== | 42% 331 MB |=========================== | 42% 331 MB |=========================== | 42% 331 MB |=========================== | 42% 331 MB |=========================== | 42% 331 MB |=========================== | 42% 331 MB |=========================== | 42% 331 MB |=========================== | 43% 331 MB |=========================== | 43% 331 MB |=========================== | 43% 331 MB |=========================== | 43% 331 MB |=========================== | 43% 331 MB |=========================== | 43% 332 MB |=========================== | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 332 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 333 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 334 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 335 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 336 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 337 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 338 MB |============================ | 43% 339 MB |============================ | 43% 339 MB |============================ | 43% 339 MB |============================ | 43% 339 MB |============================ | 44% 339 MB |============================ | 44% 339 MB |============================ | 44% 339 MB |============================ | 44% 339 MB |============================ | 44% 339 MB |============================ | 44% 339 MB |============================ | 44% 339 MB |============================ | 44% 339 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 340 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 341 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 342 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 343 MB |============================ | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 344 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 345 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 44% 346 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 347 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 348 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 349 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 350 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 351 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 352 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 353 MB |============================= | 45% 354 MB |============================= | 45% 354 MB |============================= | 45% 354 MB |============================= | 45% 354 MB |============================= | 45% 354 MB |============================= | 45% 354 MB |============================= | 45% 354 MB |============================= | 45% 354 MB |============================= | 46% 354 MB |============================= | 46% 354 MB |============================= | 46% 354 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================= | 46% 355 MB |============================== | 46% 355 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 356 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 357 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 358 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 359 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 360 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 361 MB |============================== | 46% 362 MB |============================== | 46% 362 MB |============================== | 46% 362 MB |============================== | 46% 362 MB |============================== | 46% 362 MB |============================== | 47% 362 MB |============================== | 47% 362 MB |============================== | 47% 362 MB |============================== | 47% 362 MB |============================== | 47% 362 MB |============================== | 47% 362 MB |============================== | 47% 362 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 363 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 364 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 365 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 366 MB |============================== | 47% 367 MB |============================== | 47% 367 MB |============================== | 47% 367 MB |============================== | 47% 367 MB |============================== | 47% 367 MB |============================== | 47% 367 MB |============================== | 47% 367 MB |============================== | 47% 367 MB |=============================== | 47% 367 MB |=============================== | 47% 367 MB |=============================== | 47% 367 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 368 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 369 MB |=============================== | 47% 370 MB |=============================== | 47% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 370 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 371 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 372 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 373 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 374 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 375 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 376 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 48% 377 MB |=============================== | 49% 377 MB |=============================== | 49% 377 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 378 MB |=============================== | 49% 379 MB |=============================== | 49% 379 MB |=============================== | 49% 379 MB |=============================== | 49% 379 MB |=============================== | 49% 379 MB |=============================== | 49% 379 MB |=============================== | 49% 379 MB |=============================== | 49% 379 MB |================================ | 49% 379 MB |================================ | 49% 379 MB |================================ | 49% 379 MB |================================ | 49% 379 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 380 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 381 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 382 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 383 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 384 MB |================================ | 49% 385 MB |================================ | 49% 385 MB |================================ | 49% 385 MB |================================ | 49% 385 MB |================================ | 49% 385 MB |================================ | 49% 385 MB |================================ | 50% 385 MB |================================ | 50% 385 MB |================================ | 50% 385 MB |================================ | 50% 385 MB |================================ | 50% 385 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 386 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 387 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 388 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 389 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 390 MB |================================ | 50% 391 MB |================================ | 50% 391 MB |================================ | 50% 391 MB |================================ | 50% 391 MB |================================ | 50% 391 MB |================================ | 50% 391 MB |================================= | 50% 391 MB |================================= | 50% 391 MB |================================= | 50% 391 MB |================================= | 50% 391 MB |================================= | 50% 391 MB |================================= | 50% 391 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 392 MB |================================= | 50% 393 MB |================================= | 50% 393 MB |================================= | 50% 393 MB |================================= | 51% 393 MB |================================= | 51% 393 MB |================================= | 51% 393 MB |================================= | 51% 393 MB |================================= | 51% 393 MB |================================= | 51% 393 MB |================================= | 51% 393 MB |================================= | 51% 393 MB |================================= | 51% 393 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 394 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 395 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 396 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 397 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 398 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 399 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 51% 400 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 401 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 402 MB |================================= | 52% 403 MB |================================= | 52% 403 MB |================================= | 52% 403 MB |================================= | 52% 403 MB |================================== | 52% 403 MB |================================== | 52% 403 MB |================================== | 52% 403 MB |================================== | 52% 403 MB |================================== | 52% 403 MB |================================== | 52% 403 MB |================================== | 52% 403 MB |================================== | 52% 403 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 404 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 405 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 406 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 407 MB |================================== | 52% 408 MB |================================== | 52% 408 MB |================================== | 52% 408 MB |================================== | 52% 408 MB |================================== | 52% 408 MB |================================== | 52% 408 MB |================================== | 52% 408 MB |================================== | 52% 408 MB |================================== | 53% 408 MB |================================== | 53% 408 MB |================================== | 53% 408 MB |================================== | 53% 408 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 409 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 410 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 411 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 412 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 413 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 414 MB |================================== | 53% 415 MB |================================== | 53% 415 MB |================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 415 MB |=================================== | 53% 416 MB |=================================== | 53% 416 MB |=================================== | 53% 416 MB |=================================== | 53% 416 MB |=================================== | 54% 416 MB |=================================== | 54% 416 MB |=================================== | 54% 416 MB |=================================== | 54% 416 MB |=================================== | 54% 416 MB |=================================== | 54% 416 MB |=================================== | 54% 416 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 417 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 418 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 419 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 420 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 421 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 422 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 423 MB |=================================== | 54% 424 MB |=================================== | 54% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 424 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 425 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 426 MB |=================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 427 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 428 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 429 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 430 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 55% 431 MB |==================================== | 56% 431 MB |==================================== | 56% 431 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 432 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 433 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 434 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 435 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 436 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 437 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |==================================== | 56% 438 MB |===================================== | 56% 438 MB |===================================== | 56% 439 MB |===================================== | 56% 439 MB |===================================== | 56% 439 MB |===================================== | 56% 439 MB |===================================== | 56% 439 MB |===================================== | 56% 439 MB |===================================== | 57% 439 MB |===================================== | 57% 439 MB |===================================== | 57% 439 MB |===================================== | 57% 439 MB |===================================== | 57% 439 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 440 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 441 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 442 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 443 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 444 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 445 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 446 MB |===================================== | 57% 447 MB |===================================== | 57% 447 MB |===================================== | 57% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 447 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 448 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 449 MB |===================================== | 58% 450 MB |===================================== | 58% 450 MB |===================================== | 58% 450 MB |===================================== | 58% 450 MB |===================================== | 58% 450 MB |===================================== | 58% 450 MB |===================================== | 58% 450 MB |===================================== | 58% 450 MB |===================================== | 58% 450 MB |====================================== | 58% 450 MB |====================================== | 58% 450 MB |====================================== | 58% 450 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 451 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 452 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 453 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 58% 454 MB |====================================== | 59% 454 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 455 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 456 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 457 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 458 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 459 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 460 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 461 MB |====================================== | 59% 462 MB |====================================== | 59% 462 MB |====================================== | 59% 462 MB |====================================== | 59% 462 MB |====================================== | 59% 462 MB |====================================== | 59% 462 MB |====================================== | 59% 462 MB |====================================== | 59% 462 MB |======================================= | 60% 462 MB |======================================= | 60% 462 MB |======================================= | 60% 462 MB |======================================= | 60% 462 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 463 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 464 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 465 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 466 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 467 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 468 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 469 MB |======================================= | 60% 470 MB |======================================= | 60% 470 MB |======================================= | 60% 470 MB |======================================= | 60% 470 MB |======================================= | 61% 470 MB |======================================= | 61% 470 MB |======================================= | 61% 470 MB |======================================= | 61% 470 MB |======================================= | 61% 470 MB |======================================= | 61% 470 MB |======================================= | 61% 470 MB |======================================= | 61% 470 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 471 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 472 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 473 MB |======================================= | 61% 474 MB |======================================= | 61% 474 MB |======================================= | 61% 474 MB |======================================= | 61% 474 MB |======================================= | 61% 474 MB |======================================= | 61% 474 MB |======================================== | 61% 474 MB |======================================== | 61% 474 MB |======================================== | 61% 474 MB |======================================== | 61% 474 MB |======================================== | 61% 474 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 475 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 476 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 477 MB |======================================== | 61% 478 MB |======================================== | 61% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 478 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 479 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 480 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 481 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 482 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 483 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 484 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 62% 485 MB |======================================== | 63% 485 MB |======================================== | 63% 485 MB |======================================== | 63% 486 MB |======================================== | 63% 486 MB |======================================== | 63% 486 MB |======================================== | 63% 486 MB |======================================== | 63% 486 MB |========================================= | 63% 486 MB |========================================= | 63% 486 MB |========================================= | 63% 486 MB |========================================= | 63% 486 MB |========================================= | 63% 486 MB |========================================= | 63% 486 MB |========================================= | 63% 486 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 487 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 488 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 489 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 490 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 491 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 492 MB |========================================= | 63% 493 MB |========================================= | 63% 493 MB |========================================= | 63% 493 MB |========================================= | 63% 493 MB |========================================= | 63% 493 MB |========================================= | 63% 493 MB |========================================= | 64% 493 MB |========================================= | 64% 493 MB |========================================= | 64% 493 MB |========================================= | 64% 493 MB |========================================= | 64% 493 MB |========================================= | 64% 493 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 494 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 495 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 496 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 497 MB |========================================= | 64% 498 MB |========================================= | 64% 498 MB |========================================= | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 498 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 499 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 500 MB |========================================== | 64% 501 MB |========================================== | 64% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 501 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 502 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 503 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 504 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 505 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 506 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 507 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 65% 508 MB |========================================== | 66% 508 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 509 MB |========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 510 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 511 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 512 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 513 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 514 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 515 MB |=========================================== | 66% 516 MB |=========================================== | 66% 516 MB |=========================================== | 66% 516 MB |=========================================== | 66% 516 MB |=========================================== | 66% 516 MB |=========================================== | 66% 516 MB |=========================================== | 66% 516 MB |=========================================== | 66% 516 MB |=========================================== | 67% 516 MB |=========================================== | 67% 516 MB |=========================================== | 67% 516 MB |=========================================== | 67% 516 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 517 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 518 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 519 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 520 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |=========================================== | 67% 521 MB |============================================ | 67% 521 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 522 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 523 MB |============================================ | 67% 524 MB |============================================ | 67% 524 MB |============================================ | 67% 524 MB |============================================ | 67% 524 MB |============================================ | 68% 524 MB |============================================ | 68% 524 MB |============================================ | 68% 524 MB |============================================ | 68% 524 MB |============================================ | 68% 524 MB |============================================ | 68% 524 MB |============================================ | 68% 524 MB |============================================ | 68% 524 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 525 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 526 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 527 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 528 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 529 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 530 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 531 MB |============================================ | 68% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 532 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================ | 69% 533 MB |============================================= | 69% 533 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 534 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 535 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 536 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 537 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 538 MB |============================================= | 69% 539 MB |============================================= | 69% 539 MB |============================================= | 69% 539 MB |============================================= | 69% 539 MB |============================================= | 69% 539 MB |============================================= | 69% 539 MB |============================================= | 69% 539 MB |============================================= | 69% 539 MB |============================================= | 69% 539 MB |============================================= | 70% 539 MB |============================================= | 70% 539 MB |============================================= | 70% 539 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 540 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 541 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 542 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 543 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 544 MB |============================================= | 70% 545 MB |============================================= | 70% 545 MB |============================================= | 70% 545 MB |============================================= | 70% 545 MB |============================================= | 70% 545 MB |============================================= | 70% 545 MB |============================================= | 70% 545 MB |============================================= | 70% 545 MB |============================================== | 70% 545 MB |============================================== | 70% 545 MB |============================================== | 70% 545 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 546 MB |============================================== | 70% 547 MB |============================================== | 70% 547 MB |============================================== | 70% 547 MB |============================================== | 70% 547 MB |============================================== | 70% 547 MB |============================================== | 70% 547 MB |============================================== | 71% 547 MB |============================================== | 71% 547 MB |============================================== | 71% 547 MB |============================================== | 71% 547 MB |============================================== | 71% 547 MB |============================================== | 71% 547 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 548 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 549 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 550 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 551 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 552 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 553 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 554 MB |============================================== | 71% 555 MB |============================================== | 71% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 555 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 556 MB |============================================== | 72% 557 MB |============================================== | 72% 557 MB |============================================== | 72% 557 MB |============================================== | 72% 557 MB |============================================== | 72% 557 MB |============================================== | 72% 557 MB |============================================== | 72% 557 MB |=============================================== | 72% 557 MB |=============================================== | 72% 557 MB |=============================================== | 72% 557 MB |=============================================== | 72% 557 MB |=============================================== | 72% 557 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 558 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 559 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 560 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 561 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 72% 562 MB |=============================================== | 73% 562 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 563 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 564 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 565 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 566 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 567 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 568 MB |=============================================== | 73% 569 MB |=============================================== | 73% 569 MB |=============================================== | 73% 569 MB |=============================================== | 73% 569 MB |=============================================== | 73% 569 MB |================================================ | 73% 569 MB |================================================ | 73% 569 MB |================================================ | 73% 569 MB |================================================ | 73% 569 MB |================================================ | 73% 569 MB |================================================ | 73% 569 MB |================================================ | 73% 569 MB |================================================ | 73% 570 MB |================================================ | 73% 570 MB |================================================ | 73% 570 MB |================================================ | 73% 570 MB |================================================ | 73% 570 MB |================================================ | 73% 570 MB |================================================ | 73% 570 MB |================================================ | 73% 570 MB |================================================ | 74% 570 MB |================================================ | 74% 570 MB |================================================ | 74% 570 MB |================================================ | 74% 570 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 571 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 572 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 573 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 574 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 575 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 576 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 577 MB |================================================ | 74% 578 MB |================================================ | 74% 578 MB |================================================ | 74% 578 MB |================================================ | 74% 578 MB |================================================ | 75% 578 MB |================================================ | 75% 578 MB |================================================ | 75% 578 MB |================================================ | 75% 578 MB |================================================ | 75% 578 MB |================================================ | 75% 578 MB |================================================ | 75% 578 MB |================================================ | 75% 578 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 579 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 580 MB |================================================ | 75% 581 MB |================================================ | 75% 581 MB |================================================ | 75% 581 MB |================================================ | 75% 581 MB |================================================= | 75% 581 MB |================================================= | 75% 581 MB |================================================= | 75% 581 MB |================================================= | 75% 581 MB |================================================= | 75% 581 MB |================================================= | 75% 581 MB |================================================= | 75% 581 MB |================================================= | 75% 581 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 582 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 583 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 584 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 585 MB |================================================= | 75% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 586 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 587 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 588 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 589 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 590 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 591 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 592 MB |================================================= | 76% 593 MB |================================================= | 76% 593 MB |================================================== | 76% 593 MB |================================================== | 76% 593 MB |================================================== | 76% 593 MB |================================================== | 76% 593 MB |================================================== | 76% 593 MB |================================================== | 76% 593 MB |================================================== | 76% 593 MB |================================================== | 77% 593 MB |================================================== | 77% 593 MB |================================================== | 77% 593 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 594 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 595 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 596 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 597 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 598 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 599 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 600 MB |================================================== | 77% 601 MB |================================================== | 77% 601 MB |================================================== | 77% 601 MB |================================================== | 77% 601 MB |================================================== | 77% 601 MB |================================================== | 77% 601 MB |================================================== | 78% 601 MB |================================================== | 78% 601 MB |================================================== | 78% 601 MB |================================================== | 78% 601 MB |================================================== | 78% 601 MB |================================================== | 78% 601 MB |================================================== | 78% 601 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 602 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 603 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |================================================== | 78% 604 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 605 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 606 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 607 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 608 MB |=================================================== | 78% 609 MB |=================================================== | 78% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 609 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 610 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 611 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 612 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 613 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 614 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 615 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |=================================================== | 79% 616 MB |==================================================== | 80% 616 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 617 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 618 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 619 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 620 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 621 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 622 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 623 MB |==================================================== | 80% 624 MB |==================================================== | 80% 624 MB |==================================================== | 80% 624 MB |==================================================== | 80% 624 MB |==================================================== | 80% 624 MB |==================================================== | 80% 624 MB |==================================================== | 80% 624 MB |==================================================== | 80% 624 MB |==================================================== | 81% 624 MB |==================================================== | 81% 624 MB |==================================================== | 81% 624 MB |==================================================== | 81% 624 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 625 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 626 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 627 MB |==================================================== | 81% 628 MB |==================================================== | 81% 628 MB |==================================================== | 81% 628 MB |==================================================== | 81% 628 MB |==================================================== | 81% 628 MB |==================================================== | 81% 628 MB |==================================================== | 81% 628 MB |==================================================== | 81% 628 MB |==================================================== | 81% 628 MB |===================================================== | 81% 628 MB |===================================================== | 81% 628 MB |===================================================== | 81% 628 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 629 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 630 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 631 MB |===================================================== | 81% 632 MB |===================================================== | 81% 632 MB |===================================================== | 81% 632 MB |===================================================== | 81% 632 MB |===================================================== | 82% 632 MB |===================================================== | 82% 632 MB |===================================================== | 82% 632 MB |===================================================== | 82% 632 MB |===================================================== | 82% 632 MB |===================================================== | 82% 632 MB |===================================================== | 82% 632 MB |===================================================== | 82% 632 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 633 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 634 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 635 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 636 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 637 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 638 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 82% 639 MB |===================================================== | 83% 640 MB |===================================================== | 83% 640 MB |===================================================== | 83% 640 MB |===================================================== | 83% 640 MB |===================================================== | 83% 640 MB |===================================================== | 83% 640 MB |===================================================== | 83% 640 MB |====================================================== | 83% 640 MB |====================================================== | 83% 640 MB |====================================================== | 83% 640 MB |====================================================== | 83% 640 MB |====================================================== | 83% 640 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 641 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 642 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 643 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 644 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 645 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 646 MB |====================================================== | 83% 647 MB |====================================================== | 83% 647 MB |====================================================== | 83% 647 MB |====================================================== | 83% 647 MB |====================================================== | 83% 647 MB |====================================================== | 83% 647 MB |====================================================== | 83% 647 MB |====================================================== | 83% 647 MB |====================================================== | 83% 647 MB |====================================================== | 84% 647 MB |====================================================== | 84% 647 MB |====================================================== | 84% 647 MB |====================================================== | 84% 647 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 648 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 649 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 650 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 651 MB |====================================================== | 84% 652 MB |====================================================== | 84% 652 MB |====================================================== | 84% 652 MB |====================================================== | 84% 652 MB |====================================================== | 84% 652 MB |====================================================== | 84% 652 MB |======================================================= | 84% 652 MB |======================================================= | 84% 652 MB |======================================================= | 84% 652 MB |======================================================= | 84% 652 MB |======================================================= | 84% 652 MB |======================================================= | 84% 652 MB |======================================================= | 84% 652 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 653 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 654 MB |======================================================= | 84% 655 MB |======================================================= | 84% 655 MB |======================================================= | 84% 655 MB |======================================================= | 84% 655 MB |======================================================= | 84% 655 MB |======================================================= | 84% 655 MB |======================================================= | 85% 655 MB |======================================================= | 85% 655 MB |======================================================= | 85% 655 MB |======================================================= | 85% 655 MB |======================================================= | 85% 655 MB |======================================================= | 85% 655 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 656 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 657 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 658 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 659 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 660 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 661 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 662 MB |======================================================= | 85% 663 MB |======================================================= | 85% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 663 MB |======================================================= | 86% 664 MB |======================================================= | 86% 664 MB |======================================================= | 86% 664 MB |======================================================= | 86% 664 MB |======================================================== | 86% 664 MB |======================================================== | 86% 664 MB |======================================================== | 86% 664 MB |======================================================== | 86% 664 MB |======================================================== | 86% 664 MB |======================================================== | 86% 664 MB |======================================================== | 86% 664 MB |======================================================== | 86% 664 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 665 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 666 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 667 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 668 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 669 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 86% 670 MB |======================================================== | 87% 670 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 671 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 672 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 673 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 674 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 675 MB |======================================================== | 87% 676 MB |======================================================== | 87% 676 MB |======================================================== | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 676 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 677 MB |========================================================= | 87% 678 MB |========================================================= | 87% 678 MB |========================================================= | 87% 678 MB |========================================================= | 87% 678 MB |========================================================= | 87% 678 MB |========================================================= | 87% 678 MB |========================================================= | 87% 678 MB |========================================================= | 88% 678 MB |========================================================= | 88% 678 MB |========================================================= | 88% 678 MB |========================================================= | 88% 678 MB |========================================================= | 88% 678 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 679 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 680 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 681 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 682 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 683 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 684 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 685 MB |========================================================= | 88% 686 MB |========================================================= | 88% 686 MB |========================================================= | 88% 686 MB |========================================================= | 89% 686 MB |========================================================= | 89% 686 MB |========================================================= | 89% 686 MB |========================================================= | 89% 686 MB |========================================================= | 89% 686 MB |========================================================= | 89% 686 MB |========================================================= | 89% 686 MB |========================================================= | 89% 686 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 687 MB |========================================================= | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 688 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 689 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 690 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 691 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 692 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 89% 693 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 694 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 695 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 696 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 697 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 698 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |========================================================== | 90% 699 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 700 MB |=========================================================== | 90% 701 MB |=========================================================== | 90% 701 MB |=========================================================== | 90% 701 MB |=========================================================== | 90% 701 MB |=========================================================== | 90% 701 MB |=========================================================== | 90% 701 MB |=========================================================== | 90% 701 MB |=========================================================== | 90% 701 MB |=========================================================== | 90% 701 MB |=========================================================== | 91% 701 MB |=========================================================== | 91% 701 MB |=========================================================== | 91% 701 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 702 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 703 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 704 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 705 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 706 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 707 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 708 MB |=========================================================== | 91% 709 MB |=========================================================== | 91% 709 MB |=========================================================== | 91% 709 MB |=========================================================== | 91% 709 MB |=========================================================== | 91% 709 MB |=========================================================== | 92% 709 MB |=========================================================== | 92% 709 MB |=========================================================== | 92% 709 MB |=========================================================== | 92% 709 MB |=========================================================== | 92% 709 MB |=========================================================== | 92% 709 MB |=========================================================== | 92% 709 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 710 MB |=========================================================== | 92% 711 MB |=========================================================== | 92% 711 MB |=========================================================== | 92% 711 MB |=========================================================== | 92% 711 MB |=========================================================== | 92% 711 MB |=========================================================== | 92% 711 MB |=========================================================== | 92% 711 MB |=========================================================== | 92% 711 MB |=========================================================== | 92% 711 MB |============================================================ | 92% 711 MB |============================================================ | 92% 711 MB |============================================================ | 92% 711 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 712 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 713 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 714 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 715 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 716 MB |============================================================ | 92% 717 MB |============================================================ | 92% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 717 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 718 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 719 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 720 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 721 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 722 MB |============================================================ | 93% 723 MB |============================================================ | 93% 723 MB |============================================================ | 93% 723 MB |============================================================ | 93% 723 MB |============================================================ | 93% 723 MB |============================================================ | 93% 723 MB |============================================================ | 93% 723 MB |============================================================= | 93% 723 MB |============================================================= | 93% 723 MB |============================================================= | 93% 723 MB |============================================================= | 93% 723 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 93% 724 MB |============================================================= | 94% 724 MB |============================================================= | 94% 724 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 725 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 726 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 727 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 728 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 729 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 730 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 731 MB |============================================================= | 94% 732 MB |============================================================= | 94% 732 MB |============================================================= | 94% 732 MB |============================================================= | 94% 732 MB |============================================================= | 94% 732 MB |============================================================= | 94% 732 MB |============================================================= | 94% 732 MB |============================================================= | 95% 732 MB |============================================================= | 95% 732 MB |============================================================= | 95% 732 MB |============================================================= | 95% 732 MB |============================================================= | 95% 732 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 733 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 734 MB |============================================================= | 95% 735 MB |============================================================= | 95% 735 MB |============================================================= | 95% 735 MB |============================================================= | 95% 735 MB |============================================================= | 95% 735 MB |============================================================= | 95% 735 MB |============================================================== | 95% 735 MB |============================================================== | 95% 735 MB |============================================================== | 95% 735 MB |============================================================== | 95% 735 MB |============================================================== | 95% 735 MB |============================================================== | 95% 735 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 736 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 737 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 738 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 739 MB |============================================================== | 95% 740 MB |============================================================== | 95% 740 MB |============================================================== | 95% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 740 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 741 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 742 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 743 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 744 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 745 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 746 MB |============================================================== | 96% 747 MB |============================================================== | 96% 747 MB |============================================================== | 96% 747 MB |============================================================== | 96% 747 MB |============================================================== | 96% 747 MB |=============================================================== | 96% 747 MB |=============================================================== | 96% 747 MB |=============================================================== | 96% 747 MB |=============================================================== | 96% 747 MB |=============================================================== | 96% 747 MB |=============================================================== | 96% 747 MB |=============================================================== | 96% 747 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 748 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 749 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 750 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 751 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 752 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 753 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 754 MB |=============================================================== | 97% 755 MB |=============================================================== | 97% 755 MB |=============================================================== | 97% 755 MB |=============================================================== | 97% 755 MB |=============================================================== | 97% 755 MB |=============================================================== | 97% 755 MB |=============================================================== | 97% 755 MB |=============================================================== | 97% 755 MB |=============================================================== | 98% 755 MB |=============================================================== | 98% 755 MB |=============================================================== | 98% 755 MB |=============================================================== | 98% 755 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 756 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 757 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 758 MB |=============================================================== | 98% 759 MB |=============================================================== | 98% 759 MB |=============================================================== | 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 759 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 760 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 761 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 762 MB |================================================================| 98% 763 MB |================================================================| 98% 763 MB |================================================================| 98% 763 MB |================================================================| 98% 763 MB |================================================================| 98% 763 MB |================================================================| 99% 763 MB |================================================================| 99% 763 MB |================================================================| 99% 763 MB |================================================================| 99% 763 MB |================================================================| 99% 763 MB |================================================================| 99% 763 MB |================================================================| 99% 763 MB |================================================================| 99% 763 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 764 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 765 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 766 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 767 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 768 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 769 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 770 MB |================================================================| 99% 771 MB |=================================================================| 100% 771 MB #+begin_src R :results output :session :exports both df4.mpi %>% filter(Iteration == 3) %>% filter(Value != "timste") %>% filter(grepl("MPI_", Value)) %>% group_by(Value) %>% summarize(Mean = sum(End-Start), N=n()) #+end_src #+RESULTS: #+begin_example # A tibble: 8 x 3 Value Mean N 1 MPI_Allreduce 356.860608 304794 2 MPI_Barrier 60.943087 1575 3 MPI_Comm_rank 0.000148 315 4 MPI_Comm_size 0.000251 315 5 MPI_Irecv 2.528487 1023948 6 MPI_Isend 12.640539 1023948 7 MPI_Sendrecv 329.724298 47880 8 MPI_Waitall 46.962233 92814 #+end_example ***** Plot #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1700 :height 800 :session df4.mpi %>% filter(Rank != 0) %>% filter(Iteration == 7) %>% filter(End-Start > 0.1) -> tx; tx %>% group_by(Iteration) %>% mutate(End = End - min(Start), Start = Start - min(Start)) %>% ungroup() %>% ggplot() + facet_wrap(~Iteration) + geom_rect(aes(fill=Value, xmin=Start, xmax=End, ymin=Rank, ymax=Rank+0.9)); #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure24763btH.png]] #+begin_src R :results output :session :exports both tx %>% group_by(Rank, Value, Iteration) %>% summarize(N=n()) #+end_src #+RESULTS: #+begin_example # A tibble: 1,407 x 4 # Groups: Rank, Value [?] Rank Value Iteration N 1 0 solver 150 4 2 1 nsi_matrix 150 1 3 1 solver 150 4 4 2 nsi_matrix 150 1 5 2 solver 150 4 6 3 nsi_matrix 150 1 7 3 solver 150 4 8 4 nsi_matrix 150 1 9 4 solver 150 4 10 5 nsi_matrix 150 1 # ... with 1,397 more rows #+end_example ***** Calculate Comm / Comp for this prep trace #+begin_src R :results output :session :exports both df4.mpi %>% group_by(Rank, Iteration, Platform, Nodes, NP, Partitioning, EID) %>% filter(grepl("MPI_", Value)) %>% summarize(N=n(), S=min(Start), E=max(End), Comm=sum(End-Start), Comp=(E-S)-Comm) -> df4.sum; #+end_src #+RESULTS: #+begin_src R :results output :session :exports both df4.sum #+end_src #+RESULTS: #+begin_example # A tibble: 630 x 12 # Groups: Rank, Iteration, Platform, Nodes, NP, Partitioning [?] Rank Iteration Platform Nodes NP Partitioning EID N S 1 0 1 grisou 4 63 sfc prep-4 5454 129.7846 2 0 2 grisou 4 63 sfc prep-4 4752 174.9951 3 0 3 grisou 4 63 sfc prep-4 4873 219.0275 4 0 4 grisou 4 63 sfc prep-4 4697 263.0684 5 0 5 grisou 4 63 sfc prep-4 4660 306.4143 6 0 6 grisou 4 63 sfc prep-4 4544 349.4879 7 0 7 grisou 4 63 sfc prep-4 4448 391.9349 8 0 8 grisou 4 63 sfc prep-4 4324 433.9185 9 0 9 grisou 4 63 sfc prep-4 4263 475.6839 10 0 10 grisou 4 63 sfc prep-4 3579 517.5147 # ... with 620 more rows, and 3 more variables: E , Comm , Comp #+end_example #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 500 :height 400 :session library(ggplot2); library(tidyr); df4.sum %>% filter(Rank != 0) %>% mutate(Total = Comm+Comp) %>% select(-N, -S, -E) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID) %>% ggplot(aes(x=Iteration, y=Value, group=Iteration)) + theme_bw(base_size=12) + ylim(0,NA) + geom_boxplot() + theme(legend.position="top") + facet_wrap(~Variable) #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure24763CTO.png]] ** 50-node grisou :EXP5: ** 7-node grimoire with *Infiniband* :deprecated:EXP6: *** First attempt Experiment did not work because Infiniband was not chosen by MPI. The message tells me this: #+BEGIN_EXAMPLE No OpenFabrics connection schemes reported that they were able to be used on a specific port. As such, the openib BTL (OpenFabrics support) will be disabled for this port. Local host: grimoire-4 Local device: mlx4_0 Local port: 1 CPCs attempted: udcm #+END_EXAMPLE *** Second attempt JobId 1227284 @ nancy. **** Convert to CSV I need on the $PATH: - akypuera's otf22paje (compiled using OTF2 libraries of ScoreP 3.0) - pajeng's =pj_dump= #+begin_src shell :results output export PATH=$PATH:/home/lschnorr/akypuera/b/ export PATH=$PATH:/home/lschnorr/pajeng/b/ convert() { pushd $(dirname $otf2) otf22paje traces.otf2 | pj_dump | grep ^State | cut -d, -f2,4,5,8 | sed -e "s/ //g" -e "s/MPIRank//" | gzip > traces.csv.gz popd } EDIR=exp_6_grimoire_7 # Files already converted (whose CSV size is not zero) EXISTINGFILE=$(tempfile) OTF2FILE=$(tempfile) find $EDIR -not -empty | grep csv$ | sed -e "s/.csv.gz$//" | sort > $EXISTINGFILE find $EDIR | grep otf2 | sed -e "s/.otf2$//" | sort > $OTF2FILE for otf2 in $(comm -3 $OTF2FILE $EXISTINGFILE | sed "s/$/.otf2/"); do echo $otf2 convert $otf2 done #+end_src **** Post-processing in R #+begin_src R :results output :session :exports both :tangle do-exp6.R :tangle-mode (identity #o755) #!/usr/bin/Rscript library(readr); library(dplyr); alya_scorep_trace_read <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); read_csv(filename, col_names=c("Rank", "Start", "End", "Value"), progress=TRUE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, grepl("endste", .$Value) ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum mutate(Iteration = case_when( grepl("timste", .$Value) ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = as.integer(cumsum(Iteration))) %>% ungroup() %>% # Define metadata mutate(EID = meta[2], Platform = meta[3], Nodes = meta[4], NP = meta[5], Partitioning = meta[6]); } alya_scorep_trace_iterations <- function(filename) { alya_scorep_trace_read(filename) %>% group_by(Rank, Iteration, Platform, Nodes, NP, Partitioning, EID) %>% filter(grepl("MPI_", Value)) %>% summarize(N=n(), S=min(Start), E=max(End), Comm=sum(End-Start), Comp=(E-S)-Comm); } args = commandArgs(trailingOnly=TRUE) print(args); df <- do.call("rbind", lapply(args, function(x) { alya_scorep_trace_iterations(x) })); write.csv(df, "exp6_iterations.csv"); #+end_src **** Read in R :ATTACH: :PROPERTIES: :Attachments: exp6_iterations.csv.gz :ID: 4e48b93e-3322-4cc1-9ba9-3145d1298878 :END: ***** Reading #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/4e/48b93e-3322-4cc1-9ba9-3145d1298878/exp6_iterations.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_integer(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grimoire:4180 7:4180 Min. : 97 1st Qu.: 26.00 1st Qu.: 3.0 1st Qu.: 97 Median : 52.00 Median : 5.5 Median :112 Mean : 52.02 Mean : 5.5 Mean :105 3rd Qu.: 78.00 3rd Qu.: 8.0 3rd Qu.:112 Max. :111.00 Max. :10.0 Max. :112 Partitioning EID N Start End metis:2090 Min. :6 Min. : 3579 Min. :122.2 Min. :149.6 sfc :2090 1st Qu.:6 1st Qu.:16353 1st Qu.:176.8 1st Qu.:200.4 Median :6 Median :23759 Median :243.1 Median :267.8 Mean :6 Mean :27314 Mean :240.3 Mean :264.6 3rd Qu.:6 3rd Qu.:36328 3rd Qu.:295.3 3rd Qu.:319.3 Max. :6 Max. :85081 Max. :372.1 Max. :392.9 Comm Comp Duration Min. : 0.5519 Min. : 0.0178 Min. :18.05 1st Qu.: 8.4911 1st Qu.:12.9418 1st Qu.:22.99 Median :10.0122 Median :14.4247 Median :24.01 Mean : 9.7437 Mean :14.5349 Mean :24.28 3rd Qu.:11.0630 3rd Qu.:15.8340 3rd Qu.:26.13 Max. :28.4181 Max. :27.2230 Max. :28.45 #+end_example ***** Plot #+begin_src R :results output :session :exports both df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) #+end_src #+RESULTS: #+begin_example # A tibble: 4,180 × 10 Rank Iteration Platform Nodes NP Partitioning EID Comm Comp 1 0 1 grimoire 7 97 metis 6 28.41812 0.028814 2 0 2 grimoire 7 97 metis 6 27.58274 0.024577 3 0 3 grimoire 7 97 metis 6 27.34581 0.024256 4 0 4 grimoire 7 97 metis 6 27.04863 0.023692 5 0 5 grimoire 7 97 metis 6 26.84089 0.023234 6 0 6 grimoire 7 97 metis 6 26.69917 0.022638 7 0 7 grimoire 7 97 metis 6 26.48485 0.022195 8 0 8 grimoire 7 97 metis 6 26.27733 0.021815 9 0 9 grimoire 7 97 metis 6 26.06916 0.021781 10 0 10 grimoire 7 97 metis 6 21.13847 0.017997 # ... with 4,170 more rows, and 1 more variables: Total #+end_example #+begin_src R :results output :session :exports both library(tidyr); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID); #+end_src #+RESULTS: #+begin_example # A tibble: 12,540 × 9 Rank Iteration Platform Nodes NP Partitioning EID Variable Value 1 0 1 grimoire 7 97 metis 6 Comm 28.41812 2 0 2 grimoire 7 97 metis 6 Comm 27.58274 3 0 3 grimoire 7 97 metis 6 Comm 27.34581 4 0 4 grimoire 7 97 metis 6 Comm 27.04863 5 0 5 grimoire 7 97 metis 6 Comm 26.84089 6 0 6 grimoire 7 97 metis 6 Comm 26.69917 7 0 7 grimoire 7 97 metis 6 Comm 26.48485 8 0 8 grimoire 7 97 metis 6 Comm 26.27733 9 0 9 grimoire 7 97 metis 6 Comm 26.06916 10 0 10 grimoire 7 97 metis 6 Comm 21.13847 # ... with 12,530 more rows #+end_example #+begin_src R :results output :session :exports both df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID) %>% group_by (Iteration, Platform, Nodes, NP, Partitioning, EID, Variable) %>% summarize(Mean = mean(Value), SE = 3*sd(Value)/sqrt(n()), N=n()) %>% # to check if it worked filter(NP == 97, Partitioning == "metis") #+end_src #+RESULTS: #+begin_example Source: local data frame [30 x 10] Groups: Iteration, Platform, Nodes, NP, Partitioning, EID [10] Iteration Platform Nodes NP Partitioning EID Variable Mean 1 1 grimoire 7 97 metis 6 Comm 11.77403 2 1 grimoire 7 97 metis 6 Comp 16.53532 3 1 grimoire 7 97 metis 6 Total 28.30936 4 2 grimoire 7 97 metis 6 Comm 11.32483 5 2 grimoire 7 97 metis 6 Comp 16.14359 6 2 grimoire 7 97 metis 6 Total 27.46842 7 3 grimoire 7 97 metis 6 Comm 11.09919 8 3 grimoire 7 97 metis 6 Comp 16.13198 9 3 grimoire 7 97 metis 6 Total 27.23118 10 4 grimoire 7 97 metis 6 Comm 11.01637 # ... with 20 more rows, and 2 more variables: SE , N #+end_example #+begin_src R :results output graphics :file img/exp_6-97np-metis-sfc.pdf :exports both :width 6 :height 3 :session library(ggplot2); library(tidyr); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp, Iteration = as.integer(Iteration)) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID) %>% group_by (Iteration, Platform, Nodes, NP, Partitioning, EID, Variable) %>% summarize(Mean = mean(Value), SE = 3*sd(Value)/sqrt(n()), N=n()) %>% ggplot(aes(x=Iteration, y=Mean, ymin=Mean-SE, ymax=Mean+SE, color=Partitioning)) + theme_bw(base_size=14) + ylim(0,NA) + xlim(0,10) + scale_x_continuous(breaks=seq(1:10)) + xlab("Iteration [count]") + ylab("Time [seconds]") + geom_point() + geom_errorbar(width=.3) + geom_line () + theme(legend.position=c(.78,.7), panel.margin = unit(.08, "lines"), legend.spacing = unit(.0, "lines"), legend.margin=margin(b = -.4, unit='cm'), plot.margin = unit(x = c(0, 0, 0, 0), units = "mm") ) + facet_grid(NP~Variable) #+end_src #+RESULTS: [[file:img/exp_6-97np-metis-sfc.pdf]] ***** With Arnaud #+begin_src R :results output graphics :file img/exp_6-97np_112np-metis-sfc-LB.pdf :exports both :width 6 :height 3 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID) %>% filter(Variable == "Comp", Rank != 0, Iteration != 11) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=as.factor(Rank))) + theme_bw(base_size=14) + ylim(0,30) + ylab("Computation Time [seconds]") + xlab("Iteration [count]") + # ggtitle("Each color is an MPI rank.") + geom_point(alpha=.3) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none", panel.margin = unit(.08, "lines"), legend.spacing = unit(.0, "lines"), legend.margin=margin(b = -.4, unit='cm'), plot.margin = unit(x = c(0, 0, 0, 0), units = "mm") ) + facet_grid(NP~Partitioning) #+end_src #+RESULTS: [[file:img/exp_6-97np_112np-metis-sfc-LB.pdf]] #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID) %>% filter(Variable == "Comp", Rank != 0) %>% filter(Iteration == 10) %>% ggplot(aes(x=Value, fill=Partitioning)) + theme_bw(base_size=12) + geom_histogram() + facet_grid(Partitioning~NP); #+end_src #+RESULTS: [[file:/tmp/babel-1875bQK/figure1875FAD.png]] Status: - Load balance among processes is not equal (considering only computation) - Iteration time is dominated by the slowest rank (numbers) - Such ranks also have the higher number of points as shown in fensap.par.log so we could have guessed in advance that the load balance would be a problem. - Actually, the load balancing is bad for both SFC and METIS although it is slightly better (the max load is lower) for SFC. - Communications are, as expected, more important for SFC than for METIS but when the communication network is good (e.g., ib) this becomes a problem only at large scale (probably above 512 nodes as Ricard pointed out). However, when using a commodity network (e.g, eth0), communications are problematic at smaller scale (around above 100). Therefore, no need to go at high scale to study the load balancing problem. It is already very bad at medium scale. Questions: - Do you know how to control the partitioning (of Metis and SFC), e.g., to disable communication cost, so we have a "perfect" load balance. - Do you know how to get the mapping between the average of mesh points coordinates and ranks? This could help us understanding why Metis and SFC have difficulties computing a good load balance. A question we are asking ourselves is whether the heavy loaded processes in both METIS and SFC correspond to the same region of the mesh or not. (ref the fig) ** 2-node and 4-node grimoire with Infiniband :EXP8: *** v1 :ATTACH: :PROPERTIES: :Attachments: exp_8-v1_grimoire_2.csv.gz :ID: faffa111-90b6-4e60-97f0-39475468994c :END: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/fa/ffa111-90b6-4e60-97f0-39475468994c/exp_8-v1_grimoire_2.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grimoire:640 2:640 Min. :32 1st Qu.: 7.75 1st Qu.: 3.0 1st Qu.:32 Median :15.50 Median : 5.5 Median :32 Mean :15.50 Mean : 5.5 Mean :32 3rd Qu.:23.25 3rd Qu.: 8.0 3rd Qu.:32 Max. :31.00 Max. :10.0 Max. :32 Partitioning EID Infiniband N metis:640 Length:640 Mode :logical Min. : 3579 Class :character FALSE:320 1st Qu.:11028 Mode :character TRUE :320 Median :13216 NA's :0 Mean :13343 3rd Qu.:15539 Max. :22601 Start End Comm Comp Min. :127.2 Min. :213.0 Min. : 0.2747 Min. : 0.02016 1st Qu.:298.2 1st Qu.:382.1 1st Qu.:28.2750 1st Qu.:44.78627 Median :507.0 Median :588.7 Median :33.1652 Median :47.69400 Mean :504.3 Mean :585.0 Mean :33.0316 Mean :47.60811 3rd Qu.:711.2 3rd Qu.:791.1 3rd Qu.:36.7584 3rd Qu.:52.60334 Max. :876.5 Max. :941.3 Max. :86.2371 Max. :84.82805 Duration Min. :64.25 1st Qu.:80.60 Median :82.08 Mean :80.64 3rd Qu.:83.44 Max. :86.27 #+end_example #+begin_src R :results output graphics :file img/exp_8-v1_total.png :exports both :width 500 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Total", Rank != 0, Partitioning == "metis", NP == 32) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=Infiniband)) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(Partitioning~NP) #+end_src #+RESULTS: [[file:img/exp_8-v1_total.png]] They look identical. I think both used Infiniband. I'll adapt the run alya script, run again to get v2 below. *** v2 :ATTACH: :PROPERTIES: :Attachments: exp_8-v2_grimoire_2.csv.gz :ID: 6475f529-6f29-4c6a-8d25-49c879403b6b :END: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/64/75f529-6f29-4c6a-8d25-49c879403b6b/exp_8-v2_grimoire_2.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grimoire:640 2:640 Min. :32 1st Qu.: 7.75 1st Qu.: 3.0 1st Qu.:32 Median :15.50 Median : 5.5 Median :32 Mean :15.50 Mean : 5.5 Mean :32 3rd Qu.:23.25 3rd Qu.: 8.0 3rd Qu.:32 Max. :31.00 Max. :10.0 Max. :32 Partitioning EID Infiniband N metis:640 Length:640 Mode :logical Min. : 3579 Class :character FALSE:320 1st Qu.:11028 Mode :character TRUE :320 Median :13216 NA's :0 Mean :13343 3rd Qu.:15539 Max. :22601 Start End Comm Comp Min. :126.5 Min. :213.5 Min. : 0.3249 Min. : 0.02036 1st Qu.:298.8 1st Qu.:383.4 1st Qu.:28.6923 1st Qu.:44.80856 Median :507.5 Median :589.7 Median :33.4850 Median :47.64884 Mean :505.2 Mean :586.0 Mean :33.2070 Mean :47.57824 3rd Qu.:710.9 3rd Qu.:790.4 3rd Qu.:36.8898 3rd Qu.:52.68086 Max. :882.3 Max. :946.9 Max. :86.9266 Max. :85.24896 Duration Min. :63.92 1st Qu.:80.31 Median :82.00 Mean :80.79 3rd Qu.:83.64 Max. :86.96 #+end_example #+begin_src R :results output graphics :file img/exp_8-v2_total.png :exports both :width 500 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Total", Iteration != 10, Rank != 0, Partitioning == "metis", NP == 32) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=as.factor(Infiniband))) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(Partitioning~NP) #+end_src #+RESULTS: [[file:img/exp_8-v2_total.png]] Well, it it still very similar. But now at least when Infiniband is TRUE, the total makespan of each iteration is smaller. *** v3 :ATTACH: :PROPERTIES: :Attachments: exp_8-v3_grimoire_4.csv.gz :ID: bd355764-1704-4485-bebd-5e714e76c2be :END: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/bd/355764-1704-4485-bebd-5e714e76c2be/exp_8-v3_grimoire_4.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grimoire:1280 4:1280 Min. :64 1st Qu.:15.75 1st Qu.: 3.0 1st Qu.:64 Median :31.50 Median : 5.5 Median :64 Mean :31.50 Mean : 5.5 Mean :64 3rd Qu.:47.25 3rd Qu.: 8.0 3rd Qu.:64 Max. :63.00 Max. :10.0 Max. :64 Partitioning EID Infiniband N metis:1280 Length:1280 Mode :logical Min. : 3579 Class :character FALSE:640 1st Qu.:11750 Mode :character TRUE :640 Median :14600 NA's :0 Mean :15362 3rd Qu.:18626 Max. :28849 Start End Comm Comp Min. :261.0 Min. :302.7 Min. : 0.3612 Min. : 0.03617 1st Qu.:343.8 1st Qu.:384.1 1st Qu.:13.3089 1st Qu.:21.57136 Median :443.8 Median :483.0 Median :15.2729 Median :23.21151 Mean :442.5 Mean :481.2 Mean :15.2131 Mean :23.46578 3rd Qu.:541.2 3rd Qu.:579.3 3rd Qu.:17.4194 3rd Qu.:25.69657 Max. :621.9 Max. :653.0 Max. :41.7033 Max. :40.66069 Duration Min. :30.86 1st Qu.:38.42 Median :39.29 Mean :38.68 3rd Qu.:39.86 Max. :41.76 #+end_example #+begin_src R :results output graphics :file img/exp_8-v3_total.png :exports both :width 500 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Total", Iteration != 10, Rank != 0, Partitioning == "metis", NP == 64) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=Infiniband)) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(Partitioning~NP) #+end_src #+RESULTS: [[file:img/exp_8-v3_total.png]] Same as in v3. I think Infiniband set to TRUE is already working. *** v4 :ATTACH: :PROPERTIES: :Attachments: exp_8-v4_grimoire_4.csv.gz :ID: 96a5786d-68c5-4d6c-9a54-6947b6fd5021 :END: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/96/a5786d-68c5-4d6c-9a54-6947b6fd5021/exp_8-v4_grimoire_4.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grimoire:2400 4:2400 Min. :60 1st Qu.:14.75 1st Qu.: 3.0 1st Qu.:60 Median :29.50 Median : 5.5 Median :60 Mean :29.50 Mean : 5.5 Mean :60 3rd Qu.:44.25 3rd Qu.: 8.0 3rd Qu.:60 Max. :59.00 Max. :10.0 Max. :60 Partitioning EID Infiniband N metis:1200 Length:2400 Mode :logical Min. : 3579 sfc :1200 Class :character FALSE:1200 1st Qu.:13229 Mode :character TRUE :1200 Median :22332 NA's :0 Mean :25072 3rd Qu.:33946 Max. :69461 Start End Comm Comp Min. :238.8 Min. :278.3 Min. : 0.5323 Min. : 0.03779 1st Qu.:339.3 1st Qu.:380.7 1st Qu.:10.4865 1st Qu.:23.37386 Median :431.8 Median :471.4 Median :13.5453 Median :25.22149 Mean :431.4 Mean :470.1 Mean :13.5252 Mean :25.13902 3rd Qu.:523.3 3rd Qu.:562.9 3rd Qu.:16.4835 3rd Qu.:27.24979 Max. :643.5 Max. :676.9 Max. :44.0683 Max. :42.73652 Duration Min. :28.50 1st Qu.:36.52 Median :38.28 Mean :38.66 3rd Qu.:41.78 Max. :44.13 #+end_example #+begin_src R :results output graphics :file img/exp_8-v4_total.png :exports both :width 500 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Total", Iteration != 10, Rank != 0, NP == 60) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=Infiniband)) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(NP~Partitioning) #+end_src #+RESULTS: [[file:img/exp_8-v4_total.png]] Looks like we have proof that Infiniband can be desactivated. ** 6-node grimoire infiniband check :EXP9:ATTACH: :PROPERTIES: :Attachments: exp_9-v1_grimoire_6.csv.gz :ID: e71e949a-1221-4b20-90b5-f8b9c6585810 :END: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/e7/1e949a-1221-4b20-90b5-f8b9c6585810/exp_9-v1_grimoire_6.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grimoire:3840 6:3840 Min. :96 1st Qu.:23.75 1st Qu.: 3.0 1st Qu.:96 Median :47.50 Median : 5.5 Median :96 Mean :47.50 Mean : 5.5 Mean :96 3rd Qu.:71.25 3rd Qu.: 8.0 3rd Qu.:96 Max. :95.00 Max. :10.0 Max. :96 Partitioning EID Infiniband N metis:1920 Length:3840 Mode :logical Min. : 3579 sfc :1920 Class :character FALSE:1920 1st Qu.:16320 Mode :character TRUE :1920 Median :23759 NA's :0 Mean :27008 3rd Qu.:36328 Max. :81957 Start End Comm Comp Min. :123.4 Min. :150.7 Min. : 0.3444 Min. : 0.01754 1st Qu.:190.8 1st Qu.:217.7 1st Qu.: 8.4054 1st Qu.:14.47885 Median :251.6 Median :277.3 Median : 9.7694 Median :15.58878 Mean :251.8 Mean :277.3 Mean : 9.6905 Mean :15.79769 3rd Qu.:312.4 3rd Qu.:337.4 3rd Qu.:11.4077 3rd Qu.:17.16624 Max. :384.8 Max. :405.8 Max. :27.9786 Max. :27.03608 Duration Min. :19.22 1st Qu.:24.94 Median :26.06 Mean :25.49 3rd Qu.:26.76 Max. :28.22 #+end_example #+begin_src R :results output graphics :file img/exp_9-v1_total.png :exports both :width 500 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Total", Iteration != 10, Rank != 0) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=Infiniband)) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(NP~Partitioning) #+end_src #+RESULTS: [[file:img/exp_9-v1_total.png]] ** 8-node grimoire infiniband-only after Alya modifications :EXP10: *** Read :ATTACH: :PROPERTIES: :Attachments: exp_10-v2_grimoire_8.csv.gz :ID: 15c9204a-2bde-4ee3-b166-242729fbc30d :END: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/15/c9204a-2bde-4ee3-b166-242729fbc30d/exp_10-v2_grimoire_8.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grimoire:2560 8:2560 Min. :128 1st Qu.: 31.75 1st Qu.: 3.0 1st Qu.:128 Median : 63.50 Median : 5.5 Median :128 Mean : 63.50 Mean : 5.5 Mean :128 3rd Qu.: 95.25 3rd Qu.: 8.0 3rd Qu.:128 Max. :127.00 Max. :10.0 Max. :128 Partitioning EID Infiniband N Start metis:1280 Length:2560 Mode:logical Min. : 3579 Min. :121.9 sfc :1280 Class :character TRUE:2560 1st Qu.:17937 1st Qu.:167.1 Mode :character NA's:0 Median :25074 Median :210.0 Mean :28662 Mean :210.3 3rd Qu.:37422 3rd Qu.:254.4 Max. :91329 Max. :313.9 End Comm Comp Duration Min. :139.5 Min. : 0.4235 Min. : 0.01789 Min. :13.30 1st Qu.:186.4 1st Qu.: 3.8991 1st Qu.:10.76531 1st Qu.:16.52 Median :228.6 Median : 6.6020 Median :11.80869 Median :17.29 Mean :228.5 Mean : 6.3043 Mean :11.90795 Mean :18.21 3rd Qu.:272.9 3rd Qu.: 8.5957 3rd Qu.:13.19428 3rd Qu.:20.24 Max. :330.2 Max. :21.5603 Max. :20.72990 Max. :21.59 #+end_example *** Plot Comp #+begin_src R :results output graphics :file img/exp_10-v2_comp.png :exports both :width 500 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Comp", Iteration != 10, Rank != 0) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=Infiniband)) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(NP~Partitioning) #+end_src #+RESULTS: [[file:img/exp_10-v2_comp.png]] *** Plot mean comp (per rank, all iterations) #+begin_src R :results output graphics :file img/exp_10-v2_comp_mean.png :exports both :width 600 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Comp", Iteration != 10, Rank != 0) %>% # group_by(Rank, Platform, Nodes, NP, Partitioning, EID, Infiniband) %>% # summarize(MeanComp = mean(Value), SE=3*sd(Value)/sqrt(n()), N=n()) %>% ggplot(aes(x=Rank, y=Value, color=Partitioning)) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(NP~Partitioning) #+end_src #+RESULTS: [[file:img/exp_10-v2_comp_mean.png]] *** Partition analysis **** Process log files #+begin_src shell :results output EDIR=exp_10-v2_grimoire_8 for file in $(find $EDIR | grep results | grep log$); do OUTPUT=$(dirname $file)/$(basename $file .log).csv cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f1,3,5,7,9,11,13 | uniq > $OUTPUT cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f2,4,6,8,10,12,14 >> $OUTPUT head $OUTPUT tail $OUTPUT echo done #+end_src #+RESULTS: #+begin_example Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,14976,72536,72796,2384,17,14976 2,14641,72392,72575,448,12,14641 3,22264,72647,167221,4446,16,22264 4,28129,72464,251684,7187,11,28129 5,23239,72488,178813,4341,25,23239 6,26093,72525,209615,5640,14,26093 7,22996,72391,183401,4422,10,22996 8,21190,72393,160643,3505,7,21190 9,25178,72612,201462,5195,10,25178 118,24285,72483,188018,6181,14,24285 119,26512,72604,229879,6447,15,26512 120,24865,72374,200264,5134,10,24865 121,22912,72446,173477,4266,15,22912 122,14466,72460,73072,776,11,14466 123,14046,72589,72589,944,8,14046 124,15427,72509,73961,2155,15,15427 125,21776,72610,153930,4907,13,21776 126,21662,72595,161294,4243,12,21662 127,18484,72603,111590,4500,17,18484 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,22405,85371,172276,3477,4,22405 2,22923,86049,179769,3742,5,22923 3,24316,99541,175416,3029,5,24316 4,22907,90905,170125,3170,4,22907 5,22636,86964,173194,3448,5,22636 6,22774,88416,171711,3333,6,22774 7,23564,91625,177865,3452,7,23564 8,22211,83394,173364,3612,5,22211 9,23427,89980,179685,3588,4,23427 118,19344,51553,177808,5044,6,19344 119,19050,54006,174756,4829,3,19050 120,18742,53558,169698,4652,6,18742 121,19158,51974,179487,5103,4,19158 122,19820,53397,179497,5028,8,19820 123,19357,55432,172417,4679,6,19357 124,18855,51717,176727,5006,3,18855 125,19473,52093,179803,5107,6,19473 126,18948,49074,179769,5225,4,18948 127,19037,50759,172104,4863,6,19037 #+end_example **** Load them in R #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_npoin <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[2], "_")); meta <- gsub(".dir", "", meta); read_csv(filename) %>% gather(Variable, Value, -Rank) %>% mutate(EID = meta[1], Platform = meta[2], Nodes = meta[3], NP = meta[4], Partitioning = meta[5], Infiniband = as.logical(meta[6])); } files <- list.files("exp_10-v2_grimoire_8", pattern="results_NPOIN_NELEM_NELEW_NBOUN.csv", recursive=TRUE, full.names=TRUE); files; df <- do.call("rbind", lapply(files, function(x) { read_npoin(x) })) df %>% filter(Rank == 111, Variable == "NELEW") #+end_src #+RESULTS: #+begin_example [1] "exp_10-v2_grimoire_8/10-v2_grimoire_8_128_metis_true.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [2] "exp_10-v2_grimoire_8/10-v2_grimoire_8_128_sfc_true.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) # A tibble: 2 × 9 Rank Variable Value EID Platform Nodes NP Partitioning Infiniband 1 111 NELEW 172833 10-v2 grimoire 8 128 metis TRUE 2 111 NELEW 258786 10-v2 grimoire 8 128 sfc TRUE #+end_example #+begin_src R :results output graphics :file img/exp_10-v2_partition.png :exports both :width 600 :height 800 :session df %>% ggplot(aes(x=Rank, y=Value, color=Partitioning)) + theme_bw(base_size=12) + geom_point() + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(Variable~Partitioning, scales="free_y") #+end_src #+RESULTS: [[file:img/exp_10-v2_partition.png]] ** 8-node grimoire infiniband-only before Alya modifs :EXP11: *** Read :ATTACH: :PROPERTIES: :Attachments: exp_11-v1_grimoire_8.csv.gz :ID: 11326040-b8e2-41f3-8479-c378f70864d9 :END: #+begin_src R :results output :session :exports both library(readr); library(dplyr); df <- read_csv("data/11/326040-b8e2-41f3-8479-c378f70864d9/exp_11-v1_grimoire_8.csv.gz") %>% select(-X1) %>% rename(Start = S, End = E) %>% mutate(Duration = End - Start) %>% mutate(Platform = as.factor(Platform), Partitioning = as.factor(Partitioning), Nodes = as.factor(Nodes)); df %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( X1 = col_integer(), Rank = col_integer(), Iteration = col_integer(), Platform = col_character(), Nodes = col_integer(), NP = col_integer(), Partitioning = col_character(), EID = col_character(), Infiniband = col_logical(), N = col_integer(), S = col_double(), E = col_double(), Comm = col_double(), Comp = col_double() ) Warning message: Missing column names filled in: 'X1' [1] Rank Iteration Platform Nodes NP Min. : 0.00 Min. : 1.0 grimoire:2560 8:2560 Min. :128 1st Qu.: 31.75 1st Qu.: 3.0 1st Qu.:128 Median : 63.50 Median : 5.5 Median :128 Mean : 63.50 Mean : 5.5 Mean :128 3rd Qu.: 95.25 3rd Qu.: 8.0 3rd Qu.:128 Max. :127.00 Max. :10.0 Max. :128 Partitioning EID Infiniband N Start metis:1280 Length:2560 Mode:logical Min. : 3579 Min. :121.7 sfc :1280 Class :character TRUE:2560 1st Qu.:17937 1st Qu.:168.6 Mode :character NA's:0 Median :24815 Median :217.9 Mean :28185 Mean :217.4 3rd Qu.:37093 3rd Qu.:265.8 Max. :81957 Max. :313.0 End Comm Comp Duration Min. :142.9 Min. : 0.3322 Min. : 0.02086 Min. :15.47 1st Qu.:189.2 1st Qu.: 7.0911 1st Qu.:10.84112 1st Qu.:19.75 Median :237.8 Median : 8.1973 Median :11.59094 Median :20.00 Mean :237.1 Mean : 7.9373 Mean :11.81608 Mean :19.75 3rd Qu.:285.6 3rd Qu.: 9.0958 3rd Qu.:12.73405 3rd Qu.:20.39 Max. :328.8 Max. :21.1947 Max. :20.53365 Max. :21.22 #+end_example *** Plot Comp #+begin_src R :results output graphics :file img/exp_11-v1_comp.png :exports both :width 500 :height 400 :session library(tidyr); library(ggplot2); df %>% select(-Start, -End, -N, -Duration) %>% mutate(Total = Comm+Comp) %>% gather(Variable, Value, -Rank, -Iteration, -Platform, -Nodes, -NP, -Partitioning, -EID, -Infiniband) %>% filter(Variable == "Comp", Iteration != 10, Rank != 0) %>% ggplot(aes(x=as.factor(Iteration), y=Value, color=Infiniband)) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(NP~Partitioning) #+end_src #+RESULTS: [[file:img/exp_11-v1_comp.png]] *** Partition analysis **** Process log files #+begin_src shell :results output EDIR=exp_11-v1_grimoire_8 for file in $(find $EDIR | grep results | grep log$); do OUTPUT=$(dirname $file)/$(basename $file .log).csv cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f1,3,5,7,9,11,13 | uniq > $OUTPUT cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f2,4,6,8,10,12,14 >> $OUTPUT head $OUTPUT tail $OUTPUT echo done #+end_src #+RESULTS: #+begin_example Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,32182,174168,174611,2973,16,32182 2,20919,53529,174758,5381,15,20919 3,19727,49972,175172,5020,11,19727 4,21918,69049,174054,4277,23,21918 5,21683,64345,174280,4439,14,21683 6,22205,64122,175002,4582,18,22205 7,21361,63432,174587,4436,11,21361 8,22162,75699,174459,3914,8,22162 9,21389,66339,174799,4332,8,21389 118,19736,47749,174249,5084,13,19736 119,27122,103080,174535,7852,12,27122 120,21785,59025,175145,5304,13,21785 121,20722,56163,175018,4869,12,20722 122,21269,59300,174615,4657,10,21269 123,22403,70519,174270,4135,13,22403 124,30100,145205,174682,2874,9,30100 125,31603,142203,174120,5271,15,31603 126,22843,73894,175386,4418,11,22843 127,26719,97390,174439,6359,13,26719 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,22405,85371,172276,3477,4,22405 2,22923,86049,179769,3742,5,22923 3,24316,99541,175416,3029,5,24316 4,22907,90905,170125,3170,4,22907 5,22636,86964,173194,3448,5,22636 6,22774,88416,171711,3333,6,22774 7,23564,91625,177865,3452,7,23564 8,22211,83394,173364,3612,5,22211 9,23427,89980,179685,3588,4,23427 118,19344,51553,177808,5044,6,19344 119,19050,54006,174756,4829,3,19050 120,18742,53558,169698,4652,6,18742 121,19158,51974,179487,5103,4,19158 122,19820,53397,179497,5028,8,19820 123,19357,55432,172417,4679,6,19357 124,18855,51717,176727,5006,3,18855 125,19473,52093,179803,5107,6,19473 126,18948,49074,179769,5225,4,18948 127,19037,50759,172104,4863,6,19037 #+end_example **** Load them in R #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_npoin <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[2], "_")); meta <- gsub(".dir", "", meta); read_csv(filename) %>% gather(Variable, Value, -Rank) %>% mutate(EID = meta[1], Platform = meta[2], Nodes = meta[3], NP = meta[4], Partitioning = meta[5], Infiniband = as.logical(meta[6])); } files <- list.files("exp_11-v1_grimoire_8", pattern="results_NPOIN_NELEM_NELEW_NBOUN.csv", recursive=TRUE, full.names=TRUE); files; df <- do.call("rbind", lapply(files, function(x) { read_npoin(x) })) df %>% filter(Rank == 111, Variable == "NELEW") #+end_src #+RESULTS: #+begin_example [1] "exp_11-v1_grimoire_8/11-v1_grimoire_8_128_metis_true.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [2] "exp_11-v1_grimoire_8/11-v1_grimoire_8_128_sfc_true.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) # A tibble: 2 × 9 Rank Variable Value EID Platform Nodes NP Partitioning Infiniband 1 111 NELEW 172833 11-v1 grimoire 8 128 metis TRUE 2 111 NELEW 173994 11-v1 grimoire 8 128 sfc TRUE #+end_example **** Plot #+begin_src R :results output graphics :file img/exp_11-v1_partition.png :exports both :width 600 :height 800 :session df %>% ggplot(aes(x=Rank, y=Value, color=Partitioning)) + theme_bw(base_size=12) + geom_point() + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning):as.factor(Infiniband))) + theme(legend.position="top") + facet_grid(Variable~Partitioning, scales="free_y") #+end_src #+RESULTS: [[file:img/exp_11-v1_partition.png]] ** 6-node grimoire tracing only application functions :EXP12: :PROPERTIES: :Attachments: scorep_12-v1_grimoire_6_48_metis_true_traces.csv.gz scorep_12-v1_grimoire_6_48_sfc_true_traces.csv.gz scorep_12-v1_grimoire_6_96_metis_true_traces.csv.gz scorep_12-v1_grimoire_6_96_sfc_true_traces.csv.gz :ID: 58ddccd9-95c8-4841-9c07-336a0209e264 :END: *** Which files #+name: exp_12_traces #+begin_src shell :results output org find data/ | grep scorep_12-v1 #+end_src #+RESULTS: exp_12_traces #+BEGIN_SRC org data/58/ddccd9-95c8-4841-9c07-336a0209e264/scorep_12-v1_grimoire_6_96_metis_true_traces.csv.gz data/58/ddccd9-95c8-4841-9c07-336a0209e264/scorep_12-v1_grimoire_6_48_metis_true_traces.csv.gz data/58/ddccd9-95c8-4841-9c07-336a0209e264/scorep_12-v1_grimoire_6_96_sfc_true_traces.csv.gz data/58/ddccd9-95c8-4841-9c07-336a0209e264/scorep_12-v1_grimoire_6_48_sfc_true_traces.csv.gz #+END_SRC *** Read in R :ATTACH: - =nsi_matrix= - =solver= - =nsi_inisol= (hardly seen) #+begin_src R :results output :session :exports both :var files=exp_12_traces library(readr); library(dplyr); alya_scorep_trace_read <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[4], "_")); read_csv(filename, col_names=c("Rank", "Start", "End", "Value"), progress=FALSE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( (.$Value == "timste") ~ 1, (.$Value == "endste") ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum mutate(Iteration = case_when( (.$Value == "timste") ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = cumsum(Iteration)) %>% mutate(EID = meta[2], Platform = meta[3], Nodes = meta[4], NP = meta[5], Partitioning = meta[6]); } files <- strsplit(files, "\n")[[1]] df <- do.call("rbind", lapply(files, function(x) { alya_scorep_trace_read(x) })); # # Keep only states that are interesting # df <- df %>% filter(Value %in% c("nsi_matrix", "solver", "nsi_inisol")); #+end_src #+RESULTS: #+begin_example Attaching package: ‘dplyr’ The following objects are masked from ‘package:stats’: filter, lag The following objects are masked from ‘package:base’: intersect, setdiff, setequal, union Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) #+end_example #+begin_src R :results output :session :exports both df %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Partitioning, Value) %>% summarize(NewComp = sum(End - Start)); #+end_src #+RESULTS: #+begin_example Source: local data frame [86,400 x 9] Groups: Rank, Iteration, EID, Platform, Nodes, NP, Partitioning [?] Rank Iteration EID Platform Nodes NP Partitioning Value NewComp 1 0 1 12-v1 grimoire 6 48 metis nsi_inisol 0.000013 2 0 1 12-v1 grimoire 6 48 metis nsi_matrix 0.000075 3 0 1 12-v1 grimoire 6 48 metis solver 42.482770 4 0 1 12-v1 grimoire 6 48 sfc nsi_inisol 0.000012 5 0 1 12-v1 grimoire 6 48 sfc nsi_matrix 0.000079 6 0 1 12-v1 grimoire 6 48 sfc solver 35.042236 7 0 1 12-v1 grimoire 6 96 metis nsi_inisol 0.000014 8 0 1 12-v1 grimoire 6 96 metis nsi_matrix 0.000076 9 0 1 12-v1 grimoire 6 96 metis solver 21.981336 10 0 1 12-v1 grimoire 6 96 sfc nsi_inisol 0.000012 # ... with 86,390 more rows #+end_example *** Plots #+begin_src R :results output graphics :file img/alya_nastin_solver_3_funs.png :exports both :width 600 :height 400 :session library(ggplot2); df %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Partitioning) %>% summarize(NewComp = sum(End - Start)) %>% ggplot(aes(x=Iteration, y=NewComp, color=as.factor(Rank))) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(NP~Partitioning); #+end_src #+RESULTS: [[file:img/alya_nastin_solver_3_funs.png]] #+begin_src R :results output graphics :file img/alya_nastin_solver_solver.png :exports both :width 600 :height 400 :session df %>% filter(Value == "solver") %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Value, Partitioning) %>% summarize(NewComp = sum(End - Start)) %>% ggplot(aes(x=Iteration, y=NewComp, color=as.factor(Rank))) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(NP~Partitioning); #+end_src #+RESULTS: [[file:img/alya_nastin_solver_solver.png]] #+begin_src R :results output graphics :file img/alya_nastin_solver_nsi_matrix.png :exports both :width 600 :height 400 :session df %>% filter(Value == "nsi_matrix") %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Value, Partitioning) %>% summarize(NewComp = sum(End - Start)) %>% ggplot(aes(x=Iteration, y=NewComp, color=as.factor(Rank))) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(NP~Partitioning);#, scales="free_y") #+end_src #+RESULTS: [[file:img/alya_nastin_solver_nsi_matrix.png]] #+begin_src R :results output graphics :file img/alya_nastin_solver_nsi_inisol.png :exports both :width 600 :height 400 :session df %>% filter(Value == "nsi_inisol") %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Value, Partitioning) %>% summarize(NewComp = sum(End - Start)) %>% ggplot(aes(x=Iteration, y=NewComp, color=as.factor(Rank))) + theme_bw(base_size=12) + geom_point(alpha=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(NP~Partitioning);#, scales="free_y") #+end_src #+RESULTS: [[file:img/alya_nastin_solver_nsi_inisol.png]] ** 12-node grisou (computing instrumentations, Alya modifs yes/no) :EXP13:ATTACH: :PROPERTIES: :ID: 0717e0e2-99b8-44c7-acfc-4d4ad685a2c5 :Attachments: exp_13-v1_grisou_12 :END: *** Which files #+name: exp_13_traces #+begin_src shell :results output cd data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/ find exp_13-v1_grisou_12/| grep csv.gz #+end_src #+RESULTS: exp_13_traces : exp_13-v1_grisou_12/13-v1_grisou_12_96_sfc_true_Alya.x.modif.dir/scorep_13-v1_grisou_12_96_sfc_true_Alya.x.modif/traces.csv.gz : exp_13-v1_grisou_12/13-v1_grisou_12_96_metis_true_Alya.x.modif.dir/scorep_13-v1_grisou_12_96_metis_true_Alya.x.modif/traces.csv.gz : exp_13-v1_grisou_12/13-v1_grisou_12_192_metis_true_Alya.x.dir/scorep_13-v1_grisou_12_192_metis_true_Alya.x/traces.csv.gz : exp_13-v1_grisou_12/13-v1_grisou_12_192_sfc_true_Alya.x.dir/scorep_13-v1_grisou_12_192_sfc_true_Alya.x/traces.csv.gz : exp_13-v1_grisou_12/13-v1_grisou_12_192_metis_true_Alya.x.modif.dir/scorep_13-v1_grisou_12_192_metis_true_Alya.x.modif/traces.csv.gz : exp_13-v1_grisou_12/13-v1_grisou_12_96_metis_true_Alya.x.dir/scorep_13-v1_grisou_12_96_metis_true_Alya.x/traces.csv.gz : exp_13-v1_grisou_12/13-v1_grisou_12_96_sfc_true_Alya.x.dir/scorep_13-v1_grisou_12_96_sfc_true_Alya.x/traces.csv.gz : exp_13-v1_grisou_12/13-v1_grisou_12_192_sfc_true_Alya.x.modif.dir/scorep_13-v1_grisou_12_192_sfc_true_Alya.x.modif/traces.csv.gz #+begin_src R :results output :session :exports both :var files=exp_13_traces files <- strsplit(files, "\n")[[1]] filename <- files[[1]] filename; meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); EID = meta[2] Platform = meta[3] Nodes = meta[4] NP = meta[5] Partitioning = meta[6] Infiniband = meta[7] Alya = meta[8] EID; Platform; Nodes; NP; Partitioning; Infiniband; Alya; #+end_src #+RESULTS: : [1] "exp_13-v1_grisou_12/13-v1_grisou_12_96_sfc_true_Alya.x.modif.dir/scorep_13-v1_grisou_12_96_sfc_true_Alya.x.modif/traces.csv.gz" : [1] "13-v1" : [1] "grisou" : [1] "12" : [1] "96" : [1] "sfc" : [1] "true" : [1] "Alya.x.modif" *** Read in R to =dft13= #+begin_src R :results output :session :exports both :var files=exp_13_traces library(readr); library(dplyr); alya_scorep_trace_read <- function(filename, basedir = ".") { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); fullpath <- paste0(basedir, "/", filename); read_csv(fullpath, col_names=c("Rank", "Start", "End", "Value"), progress=FALSE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( (.$Value == "timste") ~ 1, (.$Value == "endste") ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum mutate(Iteration = case_when( (.$Value == "timste") ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = cumsum(Iteration)) %>% mutate(EID = as.factor(meta[2]), Platform = as.factor(meta[3]), Nodes = as.factor(meta[4]), NP = as.factor(meta[5]), Partitioning = as.factor(meta[6]), Infiniband = as.logical(meta[7]), Alya = as.factor(meta[8])); } files <- strsplit(files, "\n")[[1]] dft13 <- do.call("rbind", lapply(files, function(x) { alya_scorep_trace_read(x, basedir="data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5") })); # # Keep only states that are interesting # dft13 <- dft13 %>% filter(Value %in% c("nsi_matrix", "solver", "nsi_inisol")); #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Warning messages: 1: In bind_rows_(x, .id) : Unequal factor levels: coercing to character 2: In bind_rows_(x, .id) : Unequal factor levels: coercing to character 3: In bind_rows_(x, .id) : Unequal factor levels: coercing to character #+end_example #+begin_src R :results output :session :exports both dft13 %>% .$NP %>% unique dft13 %>% .$Alya %>% unique dft13; dft13 %>% mutate(Partitioning=as.factor(Partitioning), NP=as.factor(NP), Alya=as.factor(Alya)) %>% summary; #+end_src #+RESULTS: #+begin_example [1] "96" "192" [1] "Alya.x.modif" "Alya.x" Source: local data frame [2,695,680 x 12] Groups: Rank [192] Rank Start End Value Iteration EID Platform Nodes NP 1 95 0.333969 0.333970 nsi_inisol 1 13-v1 grisou 12 96 2 95 0.335719 1.417964 nsi_matrix 1 13-v1 grisou 12 96 3 95 1.420657 1.978257 solver 1 13-v1 grisou 12 96 4 95 1.982556 2.796641 solver 1 13-v1 grisou 12 96 5 95 2.799346 2.991894 solver 1 13-v1 grisou 12 96 6 95 3.010278 3.864155 solver 1 13-v1 grisou 12 96 7 95 4.066358 4.066361 nsi_inisol 1 13-v1 grisou 12 96 8 95 4.066362 4.119009 solver 1 13-v1 grisou 12 96 9 95 4.888364 5.940038 nsi_matrix 1 13-v1 grisou 12 96 10 95 5.942733 6.901603 solver 1 13-v1 grisou 12 96 # ... with 2,695,670 more rows, and 3 more variables: Partitioning , # Infiniband , Alya Rank Start End Value Min. : 0.00 Min. : 0.1715 Min. : 0.1715 solver :1843200 1st Qu.: 35.75 1st Qu.: 270.0145 1st Qu.: 270.2660 nsi_inisol: 483840 Median : 71.50 Median : 529.9820 Median : 530.3756 nsi_matrix: 368640 Mean : 79.50 Mean : 572.4097 Mean : 572.7664 doiter : 0 3rd Qu.:119.25 3rd Qu.: 785.8916 3rd Qu.: 786.1151 endste : 0 Max. :191.00 Max. :1721.8589 Max. :1721.8598 nastin : 0 (Other) : 0 Iteration EID Platform Nodes NP Min. : 1.00 13-v1:2695680 grisou:2695680 12:2695680 192:1797120 1st Qu.: 21.00 96 : 898560 Median : 47.00 Mean : 47.81 3rd Qu.: 74.00 Max. :100.00 Partitioning Infiniband Alya metis:1347840 Mode:logical Alya.x :1347840 sfc :1347840 TRUE:2695680 Alya.x.modif:1347840 NA's:0 #+end_example *** Plot only "computing time" **** nastin solver 3 functions =nsi_matrix=, =solver=, =nsi_inisol= #+begin_src R :results output graphics :file img/exp_13_alya_nastin_solver_3_funs.png :exports both :width 1200 :height 400 :session library(ggplot2); dft13 %>% filter(Rank != 0) %>% filter(NP == 192, Partitioning == "sfc") %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% summarize(NewComp = sum(End - Start)) %>% ggplot(aes(x=Iteration, y=NewComp, color=as.factor(Rank))) + theme_bw(base_size=12) + geom_point() + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(Partitioning~Alya); #+end_src #+RESULTS: [[file:img/exp_13_alya_nastin_solver_3_funs.png]] **** nastin solver only =nsi_matrix= #+begin_src R :results output graphics :file img/exp_13_alya_nastin_nsi_matrix.png :exports both :width 1200 :height 400 :session library(ggplot2); dft13 %>% filter(Value == "nsi_matrix") %>% filter(Rank != 0) %>% filter(NP == 192, Partitioning == "sfc") %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% summarize(NewComp = sum(End - Start)) %>% ggplot(aes(x=Iteration, y=NewComp, color=as.factor(Rank))) + theme_bw(base_size=12) + geom_point() + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(Partitioning~Alya); #+end_src #+RESULTS: [[file:img/exp_13_alya_nastin_nsi_matrix.png]] **** nastin solver only =solver= #+begin_src R :results output graphics :file img/exp_13_alya_nastin_solver.png :exports both :width 1200 :height 400 :session library(ggplot2); dft13 %>% filter(Value == "solver") %>% filter(Rank != 0) %>% filter(NP == 192, Partitioning == "sfc") %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% summarize(NewComp = sum(End - Start)) %>% ggplot(aes(x=Iteration, y=NewComp, color=as.factor(Rank))) + theme_bw(base_size=12) + geom_point() + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(Partitioning~Alya); #+end_src #+RESULTS: [[file:img/exp_13_alya_nastin_solver.png]] **** nastin solver only =nsi_inisol= #+begin_src R :results output graphics :file img/exp_13_alya_nastin_nsi_inisol.png :exports both :width 1200 :height 400 :session library(ggplot2); dft13 %>% filter(Value == "nsi_inisol") %>% filter(Rank != 0) %>% filter(NP == 192, Partitioning == "sfc") %>% group_by(Rank, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% summarize(NewComp = sum(End - Start)) %>% ggplot(aes(x=Iteration, y=NewComp, color=as.factor(Rank))) + theme_bw(base_size=12) + geom_point() + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(Partitioning~Alya); #+end_src #+RESULTS: [[file:img/exp_13_alya_nastin_nsi_inisol.png]] **** Amount of calls to these functions along iterations #+begin_src R :results output :session :exports both dft13 %>% filter(Value == "nsi_matrix") %>% filter(Rank != 0) %>% filter(NP==192, Rank == 95, Iteration %in% c(1,100), Alya == "Alya.x") %>% mutate(Duration = End - Start) %>% select(-EID, -Platform, -Nodes, -Infiniband, -NP, -Partitioning) %>% group_by(Iteration) %>% mutate(N=n()) %>% ungroup() %>% arrange(Start, Iteration) %>% as.data.frame() #+end_src #+RESULTS: #+begin_example Rank Start End Value Iteration Alya Duration N 1 95 0.283716 0.856840 nsi_matrix 1 Alya.x 0.573124 10 2 95 0.378482 0.927219 nsi_matrix 1 Alya.x 0.548737 10 3 95 3.298924 3.875023 nsi_matrix 1 Alya.x 0.576099 10 4 95 3.320866 3.865816 nsi_matrix 1 Alya.x 0.544950 10 5 95 6.500311 7.052999 nsi_matrix 1 Alya.x 0.552688 10 6 95 6.595464 7.164909 nsi_matrix 1 Alya.x 0.569445 10 7 95 9.646956 10.194573 nsi_matrix 1 Alya.x 0.547617 10 8 95 9.782018 10.349733 nsi_matrix 1 Alya.x 0.567715 10 9 95 12.637448 13.184027 nsi_matrix 1 Alya.x 0.546579 10 10 95 12.929592 13.499010 nsi_matrix 1 Alya.x 0.569418 10 11 95 902.438214 902.987738 nsi_matrix 100 Alya.x 0.549524 6 12 95 905.091038 905.636659 nsi_matrix 100 Alya.x 0.545621 6 13 95 907.747868 908.293808 nsi_matrix 100 Alya.x 0.545940 6 14 95 914.429194 914.996252 nsi_matrix 100 Alya.x 0.567058 6 15 95 917.061845 917.634066 nsi_matrix 100 Alya.x 0.572221 6 16 95 919.813262 920.384819 nsi_matrix 100 Alya.x 0.571557 6 #+end_example There are, for =nsi_matrix= in Rank 95: - 15 calls in iteration 1 - but only 9 in iteration 100. #+begin_src R :results output :session :exports both dft13 %>% filter(Value == "solver") %>% filter(Rank != 0) %>% filter(NP == 192, Partitioning == "sfc") %>% filter(Rank == 3, Iteration %in% c(1,100), Alya == "Alya.x") %>% mutate(Duration = End-Start) %>% select(-EID, -Platform, -Nodes, -Infiniband, -NP, -Partitioning) %>% group_by(Iteration) %>% mutate(N=n()) %>% ungroup() %>% arrange(Start, Iteration) %>% as.data.frame() #+end_src #+RESULTS: #+begin_example Rank Start End Value Iteration Alya Duration N 1 3 0.806091 1.397790 solver 1 Alya.x 0.591699 25 2 3 1.400163 1.883410 solver 1 Alya.x 0.483247 25 3 3 1.884954 1.987991 solver 1 Alya.x 0.103037 25 4 3 1.995055 2.475562 solver 1 Alya.x 0.480507 25 5 3 2.558144 2.693178 solver 1 Alya.x 0.135034 25 6 3 3.821168 4.586247 solver 1 Alya.x 0.765079 25 7 3 4.588758 5.062848 solver 1 Alya.x 0.474090 25 8 3 5.064446 5.387481 solver 1 Alya.x 0.323035 25 9 3 5.394513 5.799146 solver 1 Alya.x 0.404633 25 10 3 5.881718 6.016954 solver 1 Alya.x 0.135236 25 11 3 7.130104 7.898844 solver 1 Alya.x 0.768740 25 12 3 7.901355 8.296631 solver 1 Alya.x 0.395276 25 13 3 8.298181 8.584412 solver 1 Alya.x 0.286231 25 14 3 8.591484 9.002578 solver 1 Alya.x 0.411094 25 15 3 9.085122 9.221985 solver 1 Alya.x 0.136863 25 16 3 10.303988 11.044705 solver 1 Alya.x 0.740717 25 17 3 11.047239 11.510650 solver 1 Alya.x 0.463411 25 18 3 11.512203 11.736265 solver 1 Alya.x 0.224062 25 19 3 11.743266 12.150266 solver 1 Alya.x 0.407000 25 20 3 12.232800 12.369987 solver 1 Alya.x 0.137187 25 21 3 13.452176 14.151580 solver 1 Alya.x 0.699404 25 22 3 14.154113 14.507091 solver 1 Alya.x 0.352978 25 23 3 14.508630 14.696163 solver 1 Alya.x 0.187533 25 24 3 14.703222 15.079027 solver 1 Alya.x 0.375805 25 25 3 15.161596 15.297411 solver 1 Alya.x 0.135815 25 26 3 914.954351 915.583598 solver 100 Alya.x 0.629247 15 27 3 915.586138 915.877785 solver 100 Alya.x 0.291647 15 28 3 915.879321 916.002794 solver 100 Alya.x 0.123473 15 29 3 916.009884 916.294803 solver 100 Alya.x 0.284919 15 30 3 916.381085 916.513058 solver 100 Alya.x 0.131973 15 31 3 917.587713 918.332705 solver 100 Alya.x 0.744992 15 32 3 918.335221 918.630677 solver 100 Alya.x 0.295456 15 33 3 918.632221 918.755400 solver 100 Alya.x 0.123179 15 34 3 918.762429 919.045410 solver 100 Alya.x 0.282981 15 35 3 919.131495 919.265010 solver 100 Alya.x 0.133515 15 36 3 920.334900 921.001416 solver 100 Alya.x 0.666516 15 37 3 921.003959 921.290349 solver 100 Alya.x 0.286390 15 38 3 921.291913 921.425185 solver 100 Alya.x 0.133272 15 39 3 921.432295 921.728660 solver 100 Alya.x 0.296365 15 40 3 921.815039 921.947326 solver 100 Alya.x 0.132287 15 #+end_example There are, for =solver= in Rank 95: - 25 calls in iteration 1 - but only 15 in iteration 100. Let's check for other ranks and kernels: #+begin_src R :results output :session :exports both dft13 %>% filter(Value == "solver") %>% filter(Rank != 0) %>% filter(NP == 192, Partitioning == "sfc") %>% filter(Iteration %in% c(1,100), Alya == "Alya.x") %>% mutate(Duration = End-Start) %>% select(-EID, -Platform, -Nodes, -Infiniband, -NP, -Partitioning) %>% group_by(Rank, Value, Iteration) %>% mutate(N=n()) %>% ungroup() %>% select(Rank, Iteration, N, Value) %>% unique %>% sample_n(20) %>% arrange(Iteration, Value, N) #+end_src #+RESULTS: #+begin_example # A tibble: 20 × 4 Rank Iteration N Value 1 163 1 25 solver 2 36 1 25 solver 3 129 1 25 solver 4 66 1 25 solver 5 89 1 25 solver 6 185 1 25 solver 7 97 1 25 solver 8 152 1 25 solver 9 114 1 25 solver 10 159 1 25 solver 11 94 1 25 solver 12 57 1 25 solver 13 134 1 25 solver 14 46 100 15 solver 15 101 100 15 solver 16 159 100 15 solver 17 147 100 15 solver 18 153 100 15 solver 19 124 100 15 solver 20 174 100 15 solver #+end_example #+begin_src R :results output :session :exports both dft13 %>% filter(Value == "nsi_matrix") %>% filter(Rank != 0) %>% filter(NP == 192, Partitioning == "sfc") %>% filter(Iteration %in% c(1,100), Alya == "Alya.x") %>% mutate(Duration = End-Start) %>% select(-EID, -Platform, -Nodes, -Infiniband, -NP, -Partitioning) %>% group_by(Rank, Value, Iteration) %>% mutate(N=n()) %>% ungroup() %>% select(Rank, Iteration, N, Value) %>% unique %>% sample_n(20) %>% arrange(Iteration, Value, N) #+end_src #+RESULTS: #+begin_example # A tibble: 20 × 4 Rank Iteration N Value 1 167 1 5 nsi_matrix 2 63 1 5 nsi_matrix 3 153 1 5 nsi_matrix 4 75 1 5 nsi_matrix 5 124 1 5 nsi_matrix 6 23 1 5 nsi_matrix 7 26 1 5 nsi_matrix 8 18 1 5 nsi_matrix 9 13 1 5 nsi_matrix 10 31 100 3 nsi_matrix 11 4 100 3 nsi_matrix 12 125 100 3 nsi_matrix 13 17 100 3 nsi_matrix 14 75 100 3 nsi_matrix 15 29 100 3 nsi_matrix 16 69 100 3 nsi_matrix 17 117 100 3 nsi_matrix 18 106 100 3 nsi_matrix 19 129 100 3 nsi_matrix 20 111 100 3 nsi_matrix #+end_example #+begin_src R :results output :session :exports both dft13 %>% filter(Value == "nsi_inisol") %>% filter(Rank != 0) %>% filter(NP == 192, Partitioning == "sfc") %>% filter(Iteration %in% c(1,100), Alya == "Alya.x") %>% mutate(Duration = End-Start) %>% select(-EID, -Platform, -Nodes, -Infiniband, -NP, -Partitioning) %>% group_by(Rank, Value, Iteration) %>% mutate(N=n()) %>% ungroup() %>% select(Rank, Iteration, N, Value) %>% unique %>% sample_n(20) %>% arrange(Iteration, Value, N) #+end_src #+RESULTS: #+begin_example # A tibble: 20 × 4 Rank Iteration N Value 1 27 1 6 nsi_inisol 2 171 1 6 nsi_inisol 3 155 1 6 nsi_inisol 4 166 1 6 nsi_inisol 5 134 1 6 nsi_inisol 6 137 1 6 nsi_inisol 7 45 100 4 nsi_inisol 8 110 100 4 nsi_inisol 9 7 100 4 nsi_inisol 10 9 100 4 nsi_inisol 11 89 100 4 nsi_inisol 12 171 100 4 nsi_inisol 13 190 100 4 nsi_inisol 14 16 100 4 nsi_inisol 15 30 100 4 nsi_inisol 16 186 100 4 nsi_inisol 17 32 100 4 nsi_inisol 18 10 100 4 nsi_inisol 19 23 100 4 nsi_inisol 20 146 100 4 nsi_inisol #+end_example Ok, looks like it happens for all ranks and all three kernels. *** Partition processing and reading to =dfp13= **** Process log files :deprecated: This has already been done. Not necessary to run it again. #+begin_src shell :results output EDIR=exp_13-v1_grisou_12 for file in $(find $EDIR | grep results | grep log$); do OUTPUT=$(dirname $file)/$(basename $file .log).csv cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f1,3,5,7,9,11,13 | uniq > $OUTPUT cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f2,4,6,8,10,12,14 >> $OUTPUT head $OUTPUT tail $OUTPUT echo done #+end_src #+RESULTS: #+begin_example Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,19074,96948,97208,2652,14,19074 2,22045,96927,135194,2345,14,22045 3,37420,96812,334007,9529,13,37420 4,30041,96942,231282,5441,22,30041 5,33838,96890,274890,7297,16,33838 6,29070,96827,226562,5140,9,29070 7,32347,96944,266119,6787,8,32347 8,28077,96663,216923,4865,21,28077 9,29850,96977,237277,5651,14,29850 86,37931,96789,333099,9441,13,37931 87,34578,96921,288111,9405,11,34578 88,28456,96827,200262,7317,13,28456 89,35143,97224,298674,8919,13,35143 90,32222,96838,261743,6612,14,32222 91,24232,96830,163453,3222,9,24232 92,18280,97057,97057,1365,7,18280 93,21127,97038,104990,3262,16,21127 94,29570,96812,223299,6226,11,29570 95,26541,97016,173035,6317,16,26541 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,26205,84975,235380,6679,1,26205 2,25585,77582,231837,6526,3,25585 3,25400,80246,231231,6601,1,25400 4,25438,75758,234708,6609,3,25438 5,26406,81896,239781,6312,3,26406 6,27502,95784,232364,5466,3,27502 7,26084,82696,234631,6072,2,26084 8,25678,78418,226628,6240,3,25678 9,27945,100132,233397,5326,2,27945 86,42850,239542,239542,7616,2,42850 87,32666,141054,226647,7925,4,32666 88,26385,86684,226804,5612,3,26385 89,28345,99000,239538,6482,3,28345 90,25067,72579,229554,6279,5,25067 91,26476,78672,234982,6248,6,26476 92,24732,69329,230939,6461,3,24732 93,26325,75795,237965,6480,6,26325 94,26700,84639,231454,5870,5,26700 95,27806,91670,237940,5852,3,27806 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,14777,51617,119407,2695,4,14777 2,14355,48681,119571,2829,2,14355 3,15811,59853,119398,2382,8,15811 4,13971,46640,114060,2702,5,13971 5,15956,61067,117792,2279,6,15956 6,14174,45564,119584,2966,5,14174 7,13726,42104,119489,3111,4,13726 8,14348,47398,118708,2849,5,14348 9,13241,37425,112720,3062,7,13241 182,13291,41151,119231,3125,2,13291 183,12167,34134,114984,3484,1,12167 184,12777,36965,112710,2996,3,12777 185,13270,41033,116648,3018,4,13270 186,12727,35021,112596,3165,3,12727 187,12767,37392,117632,3197,2,12767 188,12278,36271,113276,3455,1,12278 189,14272,40797,133607,3999,1,14272 190,12589,34973,112463,3186,2,12589 191,12543,35660,108215,3399,2,12543 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,22391,115758,116201,2773,17,22391 2,18665,78117,116201,2224,18,18665 3,13657,33602,116107,3330,12,13657 4,13789,33607,116562,3312,12,13789 5,13741,34479,116214,3280,11,13741 6,15414,50810,115805,2684,26,15414 7,14543,46329,115864,2777,10,14543 8,14137,34979,115739,3382,16,14137 9,14986,47013,116828,2819,12,14986 182,14122,37659,115339,3118,11,14122 183,14068,37478,115968,3182,12,14068 184,15411,45675,116180,2817,14,15411 185,15406,46701,116587,2783,15,15406 186,19875,88273,116135,2108,13,19875 187,22958,114601,116053,1817,16,22958 188,19757,84545,116610,4213,16,19757 189,14620,41964,116199,3117,12,14620 190,16129,53509,116005,2870,12,16129 191,19139,75123,115333,4732,16,19139 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,14777,51617,119407,2695,4,14777 2,14355,48681,119571,2829,2,14355 3,15811,59853,119398,2382,8,15811 4,13971,46640,114060,2702,5,13971 5,15956,61067,117792,2279,6,15956 6,14174,45564,119584,2966,5,14174 7,13726,42104,119489,3111,4,13726 8,14348,47398,118708,2849,5,14348 9,13241,37425,112720,3062,7,13241 182,13291,41151,119231,3125,2,13291 183,12167,34134,114984,3484,1,12167 184,12777,36965,112710,2996,3,12777 185,13270,41033,116648,3018,4,13270 186,12727,35021,112596,3165,3,12727 187,12767,37392,117632,3197,2,12767 188,12278,36271,113276,3455,1,12278 189,14272,40797,133607,3999,1,14272 190,12589,34973,112463,3186,2,12589 191,12543,35660,108215,3399,2,12543 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,26205,84975,235380,6679,1,26205 2,25585,77582,231837,6526,3,25585 3,25400,80246,231231,6601,1,25400 4,25438,75758,234708,6609,3,25438 5,26406,81896,239781,6312,3,26406 6,27502,95784,232364,5466,3,27502 7,26084,82696,234631,6072,2,26084 8,25678,78418,226628,6240,3,25678 9,27945,100132,233397,5326,2,27945 86,42850,239542,239542,7616,2,42850 87,32666,141054,226647,7925,4,32666 88,26385,86684,226804,5612,3,26385 89,28345,99000,239538,6482,3,28345 90,25067,72579,229554,6279,5,25067 91,26476,78672,234982,6248,6,26476 92,24732,69329,230939,6461,3,24732 93,26325,75795,237965,6480,6,26325 94,26700,84639,231454,5870,5,26700 95,27806,91670,237940,5852,3,27806 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,39242,194445,234302,5042,17,39242 2,26669,67067,232312,6639,13,26669 3,28792,85760,233695,6014,24,28792 4,28333,82323,232988,6163,12,28333 5,28519,87908,233618,5864,10,28519 6,29001,98118,233378,5377,6,29001 7,28361,89068,233513,5797,6,28361 8,30944,105897,233817,5141,21,30944 9,29093,89869,233224,5813,14,29093 86,26762,67433,233208,6686,13,26762 87,27239,68676,233356,6608,14,27239 88,26002,64284,233259,6717,10,26002 89,33558,119562,233842,9571,11,33558 90,28270,76939,233804,7049,12,28270 91,27152,74923,233598,6402,11,27152 92,29544,93864,233745,5565,11,29544 93,40608,201534,233173,3896,10,40608 94,34003,127993,232868,7401,10,34003 95,34604,129341,233472,7659,13,34604 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,9924,48191,48191,1030,17,9924 2,10099,48316,48576,1622,11,10099 3,10178,48177,48360,180,13,10178 4,12632,48021,82050,1985,14,12632 5,19592,48328,168948,4879,15,19592 6,19218,48419,165914,4699,12,19218 7,15023,48096,112771,2615,15,15023 8,15439,47989,115964,2765,26,15439 9,19266,48171,157121,4519,18,19266 182,14742,48263,110099,2721,14,14742 183,9944,48050,48662,574,12,9944 184,9674,48278,48278,674,7,9674 185,9761,48251,48251,456,10,9761 186,10670,48164,49616,1545,18,10670 187,10975,48238,54738,1717,14,10975 188,16603,48242,125207,3894,16,16603 189,13632,48288,98488,2381,12,13632 190,15115,48224,106984,3735,14,15115 191,12197,47973,64554,2533,19,12197 #+end_example **** Load them in R #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_npoin <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[5], "_")); meta <- gsub(".dir", "", meta); read_csv(filename) %>% gather(Variable, Value, -Rank) %>% mutate(EID = meta[1], Platform = meta[2], Nodes = meta[3], NP = meta[4], Partitioning = meta[5], Infiniband = as.logical(meta[6]), Alya = meta[7]); } files <- list.files("data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12", pattern="results_NPOIN_NELEM_NELEW_NBOUN.csv", recursive=TRUE, full.names=TRUE); files; dfp13 <- do.call("rbind", lapply(files, function(x) { read_npoin(x) })) dfp13 <- dfp13 %>% mutate(Partitioning=as.factor(Partitioning), NP=as.factor(NP), Alya=as.factor(Alya), Nodes=as.factor(Nodes), EID=as.factor(EID), Platform=as.factor(Platform)) dfp13 %>% filter(Rank == 111, Variable == "NELEW") #+end_src #+RESULTS: #+begin_example [1] "data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12/13-v1_grisou_12_192_metis_true_Alya.x.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [2] "data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12/13-v1_grisou_12_192_metis_true_Alya.x.modif.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [3] "data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12/13-v1_grisou_12_192_sfc_true_Alya.x.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [4] "data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12/13-v1_grisou_12_192_sfc_true_Alya.x.modif.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [5] "data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12/13-v1_grisou_12_96_metis_true_Alya.x.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [6] "data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12/13-v1_grisou_12_96_metis_true_Alya.x.modif.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [7] "data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12/13-v1_grisou_12_96_sfc_true_Alya.x.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" [8] "data/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5/exp_13-v1_grisou_12/13-v1_grisou_12_96_sfc_true_Alya.x.modif.dir/results_NPOIN_NELEM_NELEW_NBOUN.csv" Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) # A tibble: 4 × 10 Rank Variable Value EID Platform Nodes NP Partitioning Infiniband 1 111 NELEW 114133 13-v1 grisou 12 192 metis TRUE 2 111 NELEW 114133 13-v1 grisou 12 192 metis TRUE 3 111 NELEW 115711 13-v1 grisou 12 192 sfc TRUE 4 111 NELEW 101339 13-v1 grisou 12 192 sfc TRUE # ... with 1 more variables: Alya #+end_example #+begin_src R :results output :session :exports both dfp13 %>% summary #+end_src #+RESULTS: #+begin_example Rank Variable Value EID Min. : 1.00 Length:6864 Min. : 1 13-v1:6864 1st Qu.: 36.00 Class :character 1st Qu.: 3245 Median : 72.00 Mode :character Median : 15673 Mean : 80.06 Mean : 43719 3rd Qu.:120.00 3rd Qu.: 50688 Max. :191.00 Max. :335716 Platform Nodes NP Partitioning Infiniband grisou:6864 12:6864 192:4584 metis:3432 Mode:logical 96 :2280 sfc :3432 TRUE:6864 NA's:0 Alya Alya.x :3432 Alya.x.modif:3432 #+end_example *** Merge =dft13= and =dfp13= #+begin_src R :results output :session :exports both library(tidyr); dft13z <- dft13 %>% # Get all ranks except rank 0 filter(Rank != 0) %>% # Create a new variable named by the concation of the value (kernel name) and iteration mutate(Variable = Value) %>% ##paste0(Value, "-", Iteration)) %>% # Calculate the duration (put in the Value column) & remove unnecessary columns mutate(Value = End - Start) %>% select(-Start, -End) %>% select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% # Since we have several calls to our kernels for each iteration, we need to sum up group_by(Rank, Variable, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% summarize(N=n(), Value=sum(Value)) %>% ungroup() %>% # Impose an order select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Alya); dfp13z <- dfp13 %>% mutate(Iteration = NA) %>% select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Alya); dfm <- rbind(dft13z, dfp13z); dfm <- dfm %>% mutate(Iteration = as.factor(Iteration), NP = factor(NP), Partitioning = factor(Partitioning), Alya = factor(Alya), Variable = factor(Variable)); dfm %>% head; dfm %>% tail; #+end_src #+RESULTS: #+begin_example # A tibble: 6 × 10 Rank Variable Value Iteration EID Platform Nodes NP Partitioning 1 1 nsi_inisol 1.7e-05 1 13-v1 grisou 12 192 metis 2 1 nsi_inisol 1.4e-05 1 13-v1 grisou 12 192 metis 3 1 nsi_inisol 1.7e-05 1 13-v1 grisou 12 192 sfc 4 1 nsi_inisol 2.6e-05 1 13-v1 grisou 12 192 sfc 5 1 nsi_inisol 1.8e-05 1 13-v1 grisou 12 96 metis 6 1 nsi_inisol 1.9e-05 1 13-v1 grisou 12 96 metis # ... with 1 more variables: Alya # A tibble: 6 × 10 Rank Variable Value Iteration EID Platform Nodes NP Partitioning 1 90 NBBOU 32222 NA 13-v1 grisou 12 96 sfc 2 91 NBBOU 24232 NA 13-v1 grisou 12 96 sfc 3 92 NBBOU 18280 NA 13-v1 grisou 12 96 sfc 4 93 NBBOU 21127 NA 13-v1 grisou 12 96 sfc 5 94 NBBOU 29570 NA 13-v1 grisou 12 96 sfc 6 95 NBBOU 26541 NA 13-v1 grisou 12 96 sfc # ... with 1 more variables: Alya #+end_example *** Plot #+begin_src R :results output graphics :file img/exp_13_solver_partition.png :exports both :width 800 :height 600 :session library(cowplot); dfm %>% filter(Partitioning != "metis") %>% filter(NP == 192) %>% filter(is.na(Iteration) | Iteration %in% c(1,10,12, 100)) %>% filter((Variable %in% c("NELEM", "NELEW")) | grepl("solver", Variable)) %>% ggplot(aes(x=Rank, y=Value, color=Iteration)) + theme_bw(base_size=14) + geom_point() + ylim(0,NA) + theme(legend.position="top", legend.direction="horizontal", panel.margin = unit(.08, "lines"), legend.spacing = unit(.0, "lines"), legend.margin=margin(b = -.4, unit='cm'), plot.margin = unit(x = c(0, 0, 0, 0), units = "mm") ) + # ggtitle ("SFC only, NP=192, 12-nodes@grisou, Ethernet, Original (left) and modified (right) Alya, Color represents four iterations") + facet_grid(Variable~Alya, scales="free_y") -> p; save_plot("img/exp_13_solver_partition.pdf", p, base_aspect_ratio = 3.5); p; #+end_src #+RESULTS: [[file:img/exp_13_solver_partition.png]] *** During the HPC4E-report preparation meeting #+begin_src R :results output graphics :file img/exp_13_solver_partition_v2.png :exports both :width 800 :height 600 :session library(cowplot); dfm %>% filter(Partitioning != "metis") %>% filter(NP == 192) %>% filter(is.na(Iteration) | Iteration %in% c(1,10,12, 100)) %>% filter((Variable %in% c("NELEM", "NELEW", "NPOIN")) | grepl("solver", Variable)) %>% ggplot(aes(x=Rank, y=Value, color=Iteration)) + theme_bw(base_size=12) + geom_point() + ylim(0,NA) + ggtitle ("SFC only, NP=192, 12-nodes@grisou, Ethernet, Original (left) and modified (right) Alya, Color represents four iterations") + facet_grid(Variable~Alya, scales="free_y") -> p; save_plot("img/exp_13_solver_partition_v2.pdf", p, base_aspect_ratio = 3); p; #+end_src #+RESULTS: [[file:img/exp_13_solver_partition_v2.png]] *** General makespan timings #+begin_src R :results output :session :exports both dft13 %>% group_by (Partitioning, Alya) %>% summarize(Makespan=max(End)-min(Start)) #+end_src #+RESULTS: : Source: local data frame [4 x 3] : Groups: Partitioning [?] : : Partitioning Alya Makespan : : 1 metis Alya.x 1721.482 : 2 metis Alya.x.modif 1695.907 : 3 sfc Alya.x 1606.594 : 4 sfc Alya.x.modif 1382.505 ** 44-node grisou (number of non-zero entries) :EXP15: *** List files #+begin_src shell :results output find exp_15-* | grep otf2$ #+end_src #+RESULTS: : exp_15-v1_grisou_44/15-v1_grisou_44_704_metis_false_Alya.x.modif.dir/scorep_15-v1_grisou_44_704_metis_false_Alya.x.modif/traces.otf2 : exp_15-v1_grisou_44/15-v1_grisou_44_704_metis_false_Alya.x.orig.dir/scorep_15-v1_grisou_44_704_metis_false_Alya.x.orig/traces.otf2 : exp_15-v1_grisou_44/15-v1_grisou_44_704_sfc_false_Alya.x.orig.dir/scorep_15-v1_grisou_44_704_sfc_false_Alya.x.orig/traces.otf2 : exp_15-v1_grisou_44/15-v1_grisou_44_704_sfc_false_Alya.x.modif.dir/scorep_15-v1_grisou_44_704_sfc_false_Alya.x.modif/traces.otf2 *** Post-processing (convert to CSV) #+begin_src shell :results output export PATH=$PATH:~/dev/akypuera3/b/ BASEDIR=$(pwd) for directory in $(find exp_15-* | grep otf2$); do ./scripts/otf22csv_faster.sh $(dirname $directory) done #+end_src #+RESULTS: #+begin_example /home/schnorr/dev/Alya-Perf/exp_15-v1_grisou_44/15-v1_grisou_44_704_metis_false_Alya.x.modif.dir/scorep_15-v1_grisou_44_704_metis_false_Alya.x.modif/traces.otf2 ~/dev/Alya-Perf/exp_15-v1_grisou_44/15-v1_grisou_44_704_metis_false_Alya.x.modif.dir/scorep_15-v1_grisou_44_704_metis_false_Alya.x.modif ~/dev/Alya-Perf ~/dev/Alya-Perf /home/schnorr/dev/Alya-Perf/exp_15-v1_grisou_44/15-v1_grisou_44_704_metis_false_Alya.x.orig.dir/scorep_15-v1_grisou_44_704_metis_false_Alya.x.orig/traces.otf2 ~/dev/Alya-Perf/exp_15-v1_grisou_44/15-v1_grisou_44_704_metis_false_Alya.x.orig.dir/scorep_15-v1_grisou_44_704_metis_false_Alya.x.orig ~/dev/Alya-Perf ~/dev/Alya-Perf /home/schnorr/dev/Alya-Perf/exp_15-v1_grisou_44/15-v1_grisou_44_704_sfc_false_Alya.x.orig.dir/scorep_15-v1_grisou_44_704_sfc_false_Alya.x.orig/traces.otf2 ~/dev/Alya-Perf/exp_15-v1_grisou_44/15-v1_grisou_44_704_sfc_false_Alya.x.orig.dir/scorep_15-v1_grisou_44_704_sfc_false_Alya.x.orig ~/dev/Alya-Perf ~/dev/Alya-Perf /home/schnorr/dev/Alya-Perf/exp_15-v1_grisou_44/15-v1_grisou_44_704_sfc_false_Alya.x.modif.dir/scorep_15-v1_grisou_44_704_sfc_false_Alya.x.modif/traces.otf2 ~/dev/Alya-Perf/exp_15-v1_grisou_44/15-v1_grisou_44_704_sfc_false_Alya.x.modif.dir/scorep_15-v1_grisou_44_704_sfc_false_Alya.x.modif ~/dev/Alya-Perf ~/dev/Alya-Perf #+end_example #+name: exp_15_traces #+begin_src shell :results output find exp_15-* | grep traces.csv.gz$ | sort #+end_src #+RESULTS: exp_15_traces : exp_15-v1_grisou_44/15-v1_grisou_44_704_metis_false_Alya.x.modif.dir/scorep_15-v1_grisou_44_704_metis_false_Alya.x.modif/traces.csv.gz : exp_15-v1_grisou_44/15-v1_grisou_44_704_metis_false_Alya.x.orig.dir/scorep_15-v1_grisou_44_704_metis_false_Alya.x.orig/traces.csv.gz : exp_15-v1_grisou_44/15-v1_grisou_44_704_sfc_false_Alya.x.modif.dir/scorep_15-v1_grisou_44_704_sfc_false_Alya.x.modif/traces.csv.gz : exp_15-v1_grisou_44/15-v1_grisou_44_704_sfc_false_Alya.x.orig.dir/scorep_15-v1_grisou_44_704_sfc_false_Alya.x.orig/traces.csv.gz *** 1 Trace: Read and filter computation states in R to =dft15= **** Read #+name: read_exp_15_traces #+begin_src R :results output :session :exports both :var files=exp_15_traces library(readr); library(dplyr); alya_scorep_trace_read <- function(filename, basedir = ".") { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); fullpath <- paste0(basedir, "/", filename); read_delim(fullpath, delim=" ", col_names=c("Rank", "Start", "End", "Value"), progress=FALSE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( (.$Value == "timste") ~ 1, (.$Value == "endste") ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum mutate(Iteration = case_when( (.$Value == "timste") ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = cumsum(Iteration)) %>% mutate(EID = as.factor(meta[2]), Platform = as.factor(meta[3]), Nodes = as.factor(meta[4]), NP = as.factor(meta[5]), Partitioning = as.factor(meta[6]), Infiniband = as.logical(meta[7]), Alya = as.factor(meta[8])); } # Clean-up the files variable files <- strsplit(files, "\n")[[1]] # Do the reading, bind_rows everything dft15 <- do.call("bind_rows", lapply(files, function(file) { alya_scorep_trace_read(file, basedir=".") } ));#ata/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5") })); # Fix the columns factor dft15 <- dft15 %>% mutate(Partitioning = as.factor(Partitioning), Alya = as.factor(Alya)); dft15 %>% summary; #+end_src #+RESULTS: read_exp_15_traces #+begin_example Attaching package: ‘dplyr’ The following objects are masked from ‘package:stats’: filter, lag The following objects are masked from ‘package:base’: intersect, setdiff, setequal, union Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Warning messages: 1: In bind_rows_(x, .id) : Unequal factor levels: coercing to character 2: In bind_rows_(x, .id) : binding character and factor vector, coercing into character vector 3: In bind_rows_(x, .id) : binding character and factor vector, coercing into character vector 4: In bind_rows_(x, .id) : Unequal factor levels: coercing to character 5: In bind_rows_(x, .id) : binding character and factor vector, coercing into character vector 6: In bind_rows_(x, .id) : binding character and factor vector, coercing into character vector 7: In bind_rows_(x, .id) : binding character and factor vector, coercing into character vector 8: In bind_rows_(x, .id) : binding character and factor vector, coercing into character vector 9: In bind_rows_(x, .id) : binding character and factor vector, coercing into character vector Rank Start End Value Min. : 0.0 Min. : 0.0882 Min. : 0.0882 solver :8729600 1st Qu.:175.8 1st Qu.: 237.9752 1st Qu.: 238.0672 nsi_updunk:6307840 Median :351.5 Median : 449.0312 Median : 449.0313 nsi_solsgs:3491840 Mean :351.5 Mean : 465.6487 Mean : 465.7192 nsi_updbcs:2872320 3rd Qu.:527.2 3rd Qu.: 657.0840 3rd Qu.: 657.0843 nsi_inisol:2309120 Max. :703.0 Max. :1135.5267 Max. :1135.5267 nastin :2252800 (Other) :8616960 Iteration EID Platform Nodes Min. : 1.00 15-v1:34580480 grisou:34580480 44:34580480 1st Qu.: 47.00 Median : 98.00 Mean : 98.34 3rd Qu.:149.00 Max. :200.00 NP Partitioning Infiniband Alya 704:34580480 metis:17290240 Mode :logical Alya.x.modif:17290240 sfc :17290240 FALSE:34580480 Alya.x.orig :17290240 #+end_example **** Filter to get only the computation states we are interested in #+begin_src R :results output :session :exports both dft15f <- dft15 %>% filter(Value %in% c("nsi_matrix", "solver")); dft15f #+end_src #+RESULTS: #+begin_example # A tibble: 10,475,520 x 12 # Groups: Rank [704] Rank Start End Value Iteration EID Platform Nodes NP 1 0 0.116300 0.116323 nsi_matrix 1 15-v1 grisou 44 704 2 169 0.116469 0.247080 nsi_matrix 1 15-v1 grisou 44 704 3 511 0.116494 0.247081 nsi_matrix 1 15-v1 grisou 44 704 4 646 0.116509 0.248749 nsi_matrix 1 15-v1 grisou 44 704 5 391 0.116514 0.252305 nsi_matrix 1 15-v1 grisou 44 704 6 631 0.116504 0.252883 nsi_matrix 1 15-v1 grisou 44 704 7 655 0.116534 0.253021 nsi_matrix 1 15-v1 grisou 44 704 8 695 0.116470 0.253728 nsi_matrix 1 15-v1 grisou 44 704 9 374 0.116484 0.253761 nsi_matrix 1 15-v1 grisou 44 704 10 387 0.116504 0.253844 nsi_matrix 1 15-v1 grisou 44 704 # ... with 10,475,510 more rows, and 3 more variables: Partitioning , # Infiniband , Alya #+end_example *** 2 Per-rank number of elements, points, etc (process and read) The files are obtained with manual instrumentation of Alya code. **** Process log files #+begin_src shell :results output EDIR=exp_15-v1_grisou_44 for file in $(find $EDIR | grep results | grep log$ | sort); do OUTPUT=$(dirname $file)/$(basename $file .log).csv cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f1,3,5,7,9,11,13 | uniq > $OUTPUT cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f2,4,6,8,10,12,14 >> $OUTPUT head $OUTPUT tail $OUTPUT echo done #+end_src #+RESULTS: #+begin_example Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,4743,17409,32204,591,10,4743 2,4140,13120,31555,736,9,4140 3,4073,12119,32464,812,7,4073 4,5416,22804,32324,376,13,5416 5,4393,15261,30631,612,8,4393 6,4064,12109,31914,790,8,4064 7,5310,22579,32154,382,12,5310 8,4283,13578,32453,751,10,4283 9,4038,11988,31728,793,6,4038 694,3586,9252,30627,845,6,3586 695,3573,8651,30626,880,7,3573 696,3789,9266,32156,928,6,3789 697,3703,9287,31107,867,8,3703 698,4062,11473,32358,837,7,4062 699,3681,10121,32016,877,3,3681 700,3697,10030,31585,866,3,3697 701,3652,9850,30665,830,4,3652 702,3870,10305,31765,850,6,3870 703,4020,11401,32101,825,6,4020 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,4743,17409,32204,591,10,4743 2,4140,13120,31555,736,9,4140 3,4073,12119,32464,812,7,4073 4,5416,22804,32324,376,13,5416 5,4393,15261,30631,612,8,4393 6,4064,12109,31914,790,8,4064 7,5310,22579,32154,382,12,5310 8,4283,13578,32453,751,10,4283 9,4038,11988,31728,793,6,4038 694,3586,9252,30627,845,6,3586 695,3573,8651,30626,880,7,3573 696,3789,9266,32156,928,6,3789 697,3703,9287,31107,867,8,3703 698,4062,11473,32358,837,7,4062 699,3681,10121,32016,877,3,3681 700,3697,10030,31585,866,3,3697 701,3652,9850,30665,830,4,3652 702,3870,10305,31765,850,6,3870 703,4020,11401,32101,825,6,4020 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,3215,13088,13088,387,24,3215 2,3068,13094,13094,265,16,3068 3,3060,13086,13086,70,13,3060 4,3265,13092,13092,586,19,3265 5,3275,13101,13361,914,21,3275 6,3024,13106,13106,300,14,3024 7,3018,13096,13096,116,16,3018 8,2994,13100,13100,14,15,2994 9,3003,13090,13273,121,13,3003 694,3765,13097,22845,533,15,3765 695,3948,13085,23885,481,15,3948 696,4566,13108,32555,827,12,4566 697,4915,13066,33827,951,22,4915 698,4257,13110,29261,995,15,4257 699,3772,13132,19330,825,18,3772 700,4661,13103,29948,1472,20,4661 701,3588,13098,18284,1121,16,3588 702,3312,13098,13098,106,26,3312 703,3064,13096,13096,381,21,3064 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,7070,31542,31542,652,20,7070 2,7222,31267,31527,1476,20,7222 3,6680,31528,31528,524,14,6680 4,6433,31379,31562,129,13,6433 5,6734,31539,31539,77,18,6734 6,6133,24911,31560,838,16,6133 7,3800,9018,31538,906,13,3800 8,4095,8713,31453,960,16,4095 9,4041,8784,31699,909,11,4041 694,4721,12142,31657,764,21,4721 695,4360,11261,31521,958,15,4360 696,4715,14535,31538,733,15,4715 697,5309,18369,31484,659,17,5309 698,4427,12713,31430,796,13,4427 699,4600,12161,31572,901,22,4600 700,4631,14031,31540,1044,13,4631 701,5290,18228,31523,1443,14,5290 702,5418,17552,31595,1806,17,5418 703,6894,29918,31451,686,23,6894 #+end_example **** Read in R #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_npoin <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[2], "_")); meta <- gsub(".dir", "", meta); read_csv(filename) %>% gather(Variable, Value, -Rank) %>% mutate(EID = meta[1], Platform = meta[2], Nodes = meta[3], NP = meta[4], Partitioning = meta[5], Infiniband = as.logical(meta[6]), Alya = meta[7]); } files <- list.files("exp_15-v1_grisou_44", pattern="results_NPOIN_NELEM_NELEW_NBOUN.csv", recursive=TRUE, full.names=TRUE); dfp15 <- do.call("bind_rows", lapply(files, function(file) { read_npoin(file) })) dfp15 <- dfp15 %>% mutate(Partitioning=as.factor(Partitioning), NP=as.factor(NP), Alya=as.factor(Alya), Nodes=as.factor(Nodes), EID=as.factor(EID), Platform=as.factor(Platform)) dfp15 dfp15 %>% summary; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) # A tibble: 16,872 x 10 Rank Variable Value EID Platform Nodes NP Partitioning Infiniband 1 1 NPOIN 4743 15-v1 grisou 44 704 metis FALSE 2 2 NPOIN 4140 15-v1 grisou 44 704 metis FALSE 3 3 NPOIN 4073 15-v1 grisou 44 704 metis FALSE 4 4 NPOIN 5416 15-v1 grisou 44 704 metis FALSE 5 5 NPOIN 4393 15-v1 grisou 44 704 metis FALSE 6 6 NPOIN 4064 15-v1 grisou 44 704 metis FALSE 7 7 NPOIN 5310 15-v1 grisou 44 704 metis FALSE 8 8 NPOIN 4283 15-v1 grisou 44 704 metis FALSE 9 9 NPOIN 4038 15-v1 grisou 44 704 metis FALSE 10 10 NPOIN 5952 15-v1 grisou 44 704 metis FALSE # ... with 16,862 more rows, and 1 more variables: Alya Rank Variable Value EID Min. : 1 Length:16872 Min. : 0.0 15-v1:16872 1st Qu.:176 Class :character 1st Qu.: 815.8 Median :352 Mode :character Median : 4204.0 Mean :352 Mean : 9028.3 3rd Qu.:528 3rd Qu.:12905.2 Max. :703 Max. :51434.0 Platform Nodes NP Partitioning Infiniband grisou:16872 44:16872 704:16872 metis:8436 Mode :logical sfc :8436 FALSE:16872 Alya Alya.x.modif:8436 Alya.x.orig :8436 #+end_example *** 3 Per-rank number of graph entries The data is obtained by recent changes in Alya, they are in the =partition.par.post.res= file. **** Process log files To process such files, I need to extract all entries in section identified by =NUMBER_NODE_GRAPH_ENTRIES=. To do so, #+begin_src shell :results output EDIR=exp_15-v1_grisou_44 for file in $(find $EDIR | grep partition.par.post.res$ | sort); do OUTPUT=$(dirname $file)/$(basename $file .log)-entries.csv NP=$(echo $file | cut -d"/" -f2 | cut -d"_" -f4) START_LINE=$(cat -n $file | grep NUMBER_NODE_GRAPH_ENTRIES | grep ComponentNames | tr '\t' ' ' | sed "s/^[[:space:]]*//" | cut -d" " -f1) START_LINE=$(($START_LINE + 2)) END_LINE=$(($START_LINE + $NP - 1)) rm -f $OUTPUT echo "Rank,ENTRIES" >> $OUTPUT sed -n "${START_LINE},${END_LINE}p" $file | tr -s ' ' | sed "s/^ //" | tr ' ' ',' >> $OUTPUT head $OUTPUT tail $OUTPUT echo done #+end_src #+RESULTS: #+begin_example Rank,ENTRIES 1,0.675840E+05 2,0.733730E+05 3,0.675900E+05 4,0.679910E+05 5,0.792140E+05 6,0.687050E+05 7,0.672740E+05 8,0.781860E+05 9,0.697650E+05 695,0.620460E+05 696,0.618930E+05 697,0.653090E+05 698,0.634750E+05 699,0.678220E+05 700,0.642810E+05 701,0.639790E+05 702,0.626180E+05 703,0.656300E+05 704,0.672040E+05 Rank,ENTRIES 1,0.709660E+05 2,0.396250E+05 3,0.386560E+05 4,0.386280E+05 5,0.399290E+05 6,0.402150E+05 7,0.384340E+05 8,0.383040E+05 9,0.381220E+05 695,0.548130E+05 696,0.572480E+05 697,0.717460E+05 698,0.757930E+05 699,0.655330E+05 700,0.508300E+05 701,0.689990E+05 702,0.480920E+05 703,0.402860E+05 704,0.386360E+05 Rank,ENTRIES 1,0.709740E+05 2,0.910620E+05 3,0.919480E+05 4,0.884060E+05 5,0.866770E+05 6,0.886620E+05 7,0.833750E+05 8,0.646940E+05 9,0.665630E+05 695,0.720350E+05 696,0.688360E+05 697,0.718070E+05 698,0.764690E+05 699,0.694130E+05 700,0.707080E+05 701,0.709610E+05 702,0.763700E+05 703,0.771440E+05 704,0.893860E+05 #+end_example **** Read in R #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_entries <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[2], "_")); meta <- gsub(".dir", "", meta); read_csv(filename) %>% gather(Variable, Value, -Rank) %>% mutate(EID = meta[1], Platform = meta[2], Nodes = meta[3], NP = meta[4], Partitioning = meta[5], Infiniband = as.logical(meta[6]), Alya = meta[7]); } files <- list.files("exp_15-v1_grisou_44", pattern="fensap-partition.par.post.res-entries.csv", recursive=TRUE, full.names=TRUE); dfe15 <- do.call("bind_rows", lapply(files, function(file) { read_entries(file) })) dfe15 <- dfe15 %>% mutate(Partitioning=as.factor(Partitioning), NP=as.factor(NP), Alya=as.factor(Alya), Nodes=as.factor(Nodes), EID=as.factor(EID), Platform=as.factor(Platform)) %>% # Here ranks are identified starting from 1, so: # we must reduce their values by 1 to match # the trace identification mutate(Rank = Rank - 1); dfe15 dfe15 %>% summary; dfe15 %>% .$Rank %>% unique %>% sort #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Rank = col_integer(), ENTRIES = col_double() ) Parsed with column specification: cols( Rank = col_integer(), ENTRIES = col_double() ) Parsed with column specification: cols( Rank = col_integer(), ENTRIES = col_double() ) # A tibble: 2,112 x 10 Rank Variable Value EID Platform Nodes NP Partitioning Infiniband 1 0 ENTRIES 67584 15-v1 grisou 44 704 metis FALSE 2 1 ENTRIES 73373 15-v1 grisou 44 704 metis FALSE 3 2 ENTRIES 67590 15-v1 grisou 44 704 metis FALSE 4 3 ENTRIES 67991 15-v1 grisou 44 704 metis FALSE 5 4 ENTRIES 79214 15-v1 grisou 44 704 metis FALSE 6 5 ENTRIES 68705 15-v1 grisou 44 704 metis FALSE 7 6 ENTRIES 67274 15-v1 grisou 44 704 metis FALSE 8 7 ENTRIES 78186 15-v1 grisou 44 704 metis FALSE 9 8 ENTRIES 69765 15-v1 grisou 44 704 metis FALSE 10 9 ENTRIES 66860 15-v1 grisou 44 704 metis FALSE # ... with 2,102 more rows, and 1 more variables: Alya Rank Variable Value EID Platform Min. : 0.0 Length:2112 Min. : 36475 15-v1:2112 grisou:2112 1st Qu.:175.8 Class :character 1st Qu.: 65154 Median :351.5 Mode :character Median : 68318 Mean :351.5 Mean : 69842 3rd Qu.:527.2 3rd Qu.: 75119 Max. :703.0 Max. :105113 Nodes NP Partitioning Infiniband Alya 44:2112 704:2112 metis: 704 Mode :logical Alya.x.modif:1408 sfc :1408 FALSE:2112 Alya.x.orig : 704 [1] 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 [19] 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 [37] 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 [55] 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 [73] 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 [91] 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 [109] 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 [127] 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 [145] 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 [163] 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 [181] 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 [199] 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 [217] 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 [235] 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 [253] 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 [271] 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 [289] 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 [307] 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 [325] 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 [343] 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 [361] 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 [379] 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 [397] 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 [415] 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 [433] 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 [451] 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 [469] 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 [487] 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 [505] 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 [523] 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 [541] 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 [559] 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 [577] 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 [595] 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 [613] 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 [631] 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 [649] 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 [667] 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 [685] 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 [703] 702 703 #+end_example *** Merge trace and per-rank non-temporal data #+begin_src R :results output :session :exports both library(tidyr); dft15z <- dft15f %>% # Get all ranks except rank 0 filter(Rank != 0) %>% # Create a new variable named by the concation of the value (kernel name) and iteration mutate(Variable = Value) %>% ##paste0(Value, "-", Iteration)) %>% # Calculate the duration (put in the Value column) & remove unnecessary columns mutate(Value = End - Start) %>% select(-Start, -End) %>% select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% # Since we have several calls to our kernels for each iteration, we need to sum up group_by(Rank, Variable, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% summarize(N=n(), Value=sum(Value)) %>% ungroup() %>% # Impose an order select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Alya); dfp15z <- dfp15 %>% mutate(Iteration = NA) %>% select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Alya); dfe15z <- dfe15 %>% mutate(Iteration = NA) %>% select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Alya); dfm15 <- bind_rows(dft15z, dfp15z, dfe15z); dfm15 <- dfm15 %>% mutate(Iteration = as.factor(Iteration), NP = factor(NP), Partitioning = factor(Partitioning), Alya = factor(Alya), Variable = factor(Variable)); dfm15 %>% summary dfm15 %>% .$Variable %>% unique #+end_src #+RESULTS: #+begin_example Warning messages: 1: In bind_rows_(x, .id) : binding factor and character vector, coercing into character vector 2: In bind_rows_(x, .id) : binding character and factor vector, coercing into character vector Rank Variable Value Iteration Min. : 0 nsi_matrix:562400 Min. : 0.00 1 : 5624 1st Qu.:176 solver :562400 1st Qu.: 0.48 2 : 5624 Median :352 NBBOU : 2812 Median : 1.89 3 : 5624 Mean :352 NBOUN : 2812 Mean : 264.05 4 : 5624 3rd Qu.:528 NELEM : 2812 3rd Qu.: 3.70 5 : 5624 Max. :703 NELEW : 2812 Max. :105113.00 (Other):1096680 (Other) : 7736 NA's : 18984 EID Platform Nodes NP Partitioning 15-v1:1143784 grisou:1143784 44:1143784 704:1143784 metis:571540 sfc :572244 Alya Alya.x.modif:572244 Alya.x.orig :571540 [1] nsi_matrix solver NPOIN NELEM NELEW NBOUN NNEIG [8] NBBOU ENTRIES Levels: ENTRIES NBBOU NBOUN NELEM NELEW NNEIG NPOIN nsi_matrix solver #+end_example *** Plot the correlation between computaton time and NELEM, NELEW, ENTRIES #+begin_src R :results output graphics :file img/exp_15_solver_partition_entries.png :exports both :width 800 :height 600 :session library(cowplot); dfm15 %>% filter(Partitioning != "metis") %>% filter(is.na(Iteration) | Iteration %in% c(50, 100, 150, 200)) %>% filter((Variable %in% c("NELEM", "NELEW", "ENTRIES")) | grepl("solver", Variable)) -> dfm15sel; title = paste("Experiment:", (dfm15sel %>% .$EID %>% unique), (dfm15sel %>% .$Platform %>% unique), (dfm15sel %>% .$Nodes %>% unique), (dfm15sel %>% .$NP %>% unique), (dfm15sel %>% .$Partitioning %>% unique), sep = " "); dfm15sel %>% ggplot(aes(x=Rank, y=Value, color=Iteration)) + theme_bw(base_size=16) + geom_point(size=.3) + # ylim(0,NA) + theme(legend.position="top", legend.direction="horizontal", panel.margin = unit(.08, "lines"), legend.spacing = unit(.0, "lines"), legend.margin=margin(b = -.4, unit='cm'), plot.margin = unit(x = c(0, 0, 0, 0), units = "mm")) + # ggtitle (title) + facet_grid(Variable~Alya, scales="free_y") -> p; save_plot("img/exp_15_solver_partition_entries.pdf", p, base_aspect_ratio = 1.6); p; #+end_src #+RESULTS: [[file:img/exp_15_solver_partition_entries.png]] *** Analysis with Arnaud **** Solver time as a function of NELEW #+begin_src R :results output graphics :file img/exp_15_solver_as_nelew.png :exports both :width 600 :height 400 :session dfm15sel %>% filter(Rank != 0) %>% filter(Partitioning == "sfc", Iteration == 150 | is.na(Iteration)) %>% select(-Iteration) %>% spread(Variable, Value) %>% ggplot(aes(x=NELEW, y=solver, color=Rank)) + geom_point() + facet_wrap(~Alya); #+end_src #+RESULTS: [[file:img/exp_15_solver_as_nelew.png]] **** Solver time as a function of ENTRIES #+begin_src R :results output graphics :file img/exp_15_solver_as_entries.png :exports both :width 600 :height 400 :session dfm15sel %>% filter(Rank != 0) %>% filter(Partitioning == "sfc", Iteration == 100 | is.na(Iteration)) %>% select(-Iteration) %>% spread(Variable, Value) %>% ggplot(aes(x=ENTRIES, y=solver, color=Rank)) + geom_point() + facet_wrap(~Alya); #+end_src #+RESULTS: [[file:img/exp_15_solver_as_entries.png]] **** How much time one iteration takes #+begin_src R :results output :session :exports both dft15 %>% group_by(EID, Platform, Nodes, NP, Infiniband, Alya, Partitioning) %>% summarize(N=n(), Makespan = max(End) - min(Start), NBIter = max(Iteration)) %>% as.data.frame() #+end_src #+RESULTS: #+begin_example EID Platform Nodes NP Infiniband Alya Partitioning N 1 15-v1 grisou 44 704 FALSE Alya.x.modif metis 8645120 2 15-v1 grisou 44 704 FALSE Alya.x.modif sfc 8645120 3 15-v1 grisou 44 704 FALSE Alya.x.orig metis 8645120 4 15-v1 grisou 44 704 FALSE Alya.x.orig sfc 8645120 Makespan NBIter 1 721.8956 200 2 1058.6879 200 3 723.8556 200 4 1135.4094 200 #+end_example **** How many calls to "solver" in iteration #+begin_src R :results output :session :exports both dft15 %>% filter(Value == "solver") %>% filter(Iteration == 150) %>% filter(Partitioning == "sfc") %>% group_by(Rank, EID, Platform, Nodes, NP, Infiniband, Alya, Partitioning) %>% summarize(N=n()) %>% filter(grepl("modif", Alya)) %>% .$N %>% unique %>% sort # summary #+end_src #+RESULTS: : [1] 15 *** Plot iteration solver computation time along time #+begin_src R :results output graphics :file img/exp_15_solver_as_iteration.png :exports both :width 1200 :height 400 :session library(ggplot2); dfm15 %>% filter(Variable == "solver") %>% ggplot(aes(x=as.integer(Iteration), y=Value, color=as.factor(Rank))) + theme_bw(base_size=16) + geom_point(alpha=.1, size=.3) + ylim(0,NA) + geom_line (aes(group=as.factor(Rank):as.factor(Partitioning))) + theme(legend.position="none") + facet_grid(NP~Partitioning); #+end_src #+RESULTS: [[file:img/exp_15_solver_as_iteration.png]] *** Hilbert Load Curve (@BSC) #+begin_src R :results output :session :exports both dfm15 %>% filter(Iteration == 130 | is.na(Iteration)) %>% filter(Partitioning == "sfc", Alya == "Alya.x.modif") -> dfm15.sel; dfm15.sel %>% # filter(Variable == "solver") %>% filter(Iteration == 130) %>% group_by(Rank, Iteration, Partitioning, EID, Platform, Nodes, NP) %>% summarize(Computing = sum(Value)) %>% ungroup() %>% select(-Iteration) %>% arrange(Rank) -> dfm15.time; dfm15.sel %>% filter(is.na(Iteration)) %>% select(-Iteration) %>% arrange(Rank) %>% filter(Variable == "NPOIN") -> dfm15.point; #+end_src #+RESULTS: #+begin_src R :results output :session :exports both dfm15.time #+end_src #+RESULTS: #+begin_example # A tibble: 703 x 7 Rank Partitioning EID Platform Nodes NP Computing 1 1 sfc 15-v1 grisou 44 704 4.348808 2 2 sfc 15-v1 grisou 44 704 4.342806 3 3 sfc 15-v1 grisou 44 704 4.338238 4 4 sfc 15-v1 grisou 44 704 4.349558 5 5 sfc 15-v1 grisou 44 704 4.348942 6 6 sfc 15-v1 grisou 44 704 4.342368 7 7 sfc 15-v1 grisou 44 704 4.343007 8 8 sfc 15-v1 grisou 44 704 4.349633 9 9 sfc 15-v1 grisou 44 704 4.349449 10 10 sfc 15-v1 grisou 44 704 4.342640 # ... with 693 more rows #+end_example #+begin_src R :results output :session :exports both dfm15.time %>% left_join(dfm15.point %>% rename(Npoint = Value)) %>% select(-Partitioning, -EID, -Platform, -Alya, -Variable) -> dfm15.2; dfm15.2; #+end_src #+RESULTS: #+begin_example Joining, by = c("Rank", "Partitioning", "EID", "Platform", "Nodes", "NP") # A tibble: 703 x 5 Rank Nodes NP Computing Npoint 1 1 44 704 4.348808 3215 2 2 44 704 4.342806 3068 3 3 44 704 4.338238 3060 4 4 44 704 4.349558 3265 5 5 44 704 4.348942 3275 6 6 44 704 4.342368 3024 7 7 44 704 4.343007 3018 8 8 44 704 4.349633 2994 9 9 44 704 4.349449 3003 10 10 44 704 4.342640 3093 # ... with 693 more rows #+end_example #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session m1=.85 m2=m1+.05 dfm15.2 %>% arrange(Rank) %>% mutate(Computing.2 = cumsum(Computing)/sum(Computing), Npoint.2 = cumsum(Npoint)/sum(Npoint)) %>% ggplot(aes(x=Npoint.2, y=Computing.2)) + geom_point(size=2) + geom_line() + ylim(m1,m2) + xlim(m1,m2) + geom_hline(yintercept=(0:703)/703) # + geom_vline(xintercept=(0:703)/703) #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure24763_IF.png]] #+begin_src R :results output :session :exports both dfm15.sel %>% pull(Variable) %>% unique #summary(dfm15.sel) %>% #+end_src #+RESULTS: : [1] nsi_matrix solver NPOIN NELEM NELEW NBOUN NNEIG : [8] NBBOU ENTRIES : Levels: ENTRIES NBBOU NBOUN NELEM NELEW NNEIG NPOIN nsi_matrix solver *** Gantt-chart #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1700 :height 400 :session dft15 %>% filter(End-Start > 0.1) %>% filter(Partitioning == "sfc", Alya == "Alya.x.modif") %>% filter(Iteration %in% c(7)) %>%#, 151, 152, 153)) %>% filter(End < 792) -> tx; tx %>% ggplot() + geom_rect(aes(fill=Value, xmin=Start, xmax=End, ymin=Rank, ymax=Rank+0.9)); #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure24763cZy.png]] #+begin_src R :results output :session :exports both tx %>% group_by(Rank, Value, Iteration) %>% summarize(N=n()) #+end_src #+RESULTS: #+begin_example # A tibble: 2,111 x 4 # Groups: Rank, Value [?] Rank Value Iteration N 1 0 nsi_solsgs 7 2 2 0 solver 7 20 3 1 nsi_matrix 7 5 4 1 nsi_solsgs 7 2 5 1 solver 7 20 6 2 nsi_matrix 7 5 7 2 nsi_solsgs 7 2 8 2 solver 7 20 9 3 nsi_matrix 7 5 10 3 nsi_solsgs 7 2 # ... with 2,101 more rows #+end_example *** Gantt-chart replay #+begin_src R :results output :session :exports both dft15 %>% pull(Value) %>% unique #+end_src #+RESULTS: : [1] timste nsi_updunk nsi_updbcs nsi_begste nastin nsi_inisol : [7] nsi_ifconf nsi_solsgs nsi_matrix solver nsi_solite nsi_doiter : [13] doiter nsi_concou nsi_endste : 21 Levels: doiter endste nastin nsi_begste nsi_concou nsi_doiter ... timste #+begin_src R :results output graphics :file img/exp_15_gantt_chart_Alya.x.modif_iteration_7.png :exports both :width 800 :height 400 :session library(ggplot2); dft15 %>% filter(End-Start > 0.1) %>% filter(Partitioning == "sfc", Alya == "Alya.x.modif") %>% filter(Iteration %in% c(7)) %>%#, 151, 152, 153)) %>% filter(End < 792) -> tx; tx %>% ggplot() + geom_rect(aes(fill=Value, xmin=Start, xmax=End, ymin=Rank, ymax=Rank+0.9)); #+end_src #+RESULTS: [[file:/tmp/babel-10163Wik/figure10163A6o.png]] ** 8-node grimoire :EXP16: *** List files #+begin_src shell :results output find exp_16-* | grep otf2$ #+end_src #+RESULTS: : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_false_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_metis_false_Alya.x.orig/traces.otf2 : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_true_Alya.x.modif.dir/scorep_16-v1_grimoire_8_128_metis_true_Alya.x.modif/traces.otf2 : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_sfc_true_Alya.x.modif.dir/scorep_16-v1_grimoire_8_128_sfc_true_Alya.x.modif/traces.otf2 : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_true_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_metis_true_Alya.x.orig/traces.otf2 : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_sfc_true_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_sfc_true_Alya.x.orig/traces.otf2 *** Post-processing (convert to CSV) #+begin_src shell :results output export PATH=$PATH:~/dev/akypuera3/b/ BASEDIR=$(pwd) for directory in $(find exp_16-* | grep otf2$); do ./scripts/otf22csv_faster.sh $(dirname $directory) done #+end_src #+RESULTS: #+begin_example /home/schnorr/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_false_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_metis_false_Alya.x.orig/traces.otf2 ~/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_false_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_metis_false_Alya.x.orig ~/dev/Alya-Perf ~/dev/Alya-Perf /home/schnorr/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_true_Alya.x.modif.dir/scorep_16-v1_grimoire_8_128_metis_true_Alya.x.modif/traces.otf2 ~/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_true_Alya.x.modif.dir/scorep_16-v1_grimoire_8_128_metis_true_Alya.x.modif ~/dev/Alya-Perf ~/dev/Alya-Perf /home/schnorr/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_sfc_true_Alya.x.modif.dir/scorep_16-v1_grimoire_8_128_sfc_true_Alya.x.modif/traces.otf2 ~/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_sfc_true_Alya.x.modif.dir/scorep_16-v1_grimoire_8_128_sfc_true_Alya.x.modif ~/dev/Alya-Perf ~/dev/Alya-Perf /home/schnorr/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_true_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_metis_true_Alya.x.orig/traces.otf2 ~/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_true_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_metis_true_Alya.x.orig ~/dev/Alya-Perf ~/dev/Alya-Perf /home/schnorr/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_sfc_true_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_sfc_true_Alya.x.orig/traces.otf2 ~/dev/Alya-Perf/exp_16-v1_grimoire_8/16-v1_grimoire_8_128_sfc_true_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_sfc_true_Alya.x.orig ~/dev/Alya-Perf ~/dev/Alya-Perf #+end_example #+name: exp_16_traces #+begin_src shell :results output find exp_16-* | grep traces.csv.gz$ #+end_src #+RESULTS: exp_16_traces : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_false_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_metis_false_Alya.x.orig/traces.csv.gz : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_true_Alya.x.modif.dir/scorep_16-v1_grimoire_8_128_metis_true_Alya.x.modif/traces.csv.gz : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_sfc_true_Alya.x.modif.dir/scorep_16-v1_grimoire_8_128_sfc_true_Alya.x.modif/traces.csv.gz : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_metis_true_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_metis_true_Alya.x.orig/traces.csv.gz : exp_16-v1_grimoire_8/16-v1_grimoire_8_128_sfc_true_Alya.x.orig.dir/scorep_16-v1_grimoire_8_128_sfc_true_Alya.x.orig/traces.csv.gz *** 1 Trace: Read computation states in R to =dft16= **** Read #+name: read_exp_16_traces #+begin_src R :results output :session :exports both :var files=exp_16_traces library(readr); library(dplyr); alya_scorep_trace_read <- function(filename, basedir = ".") { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[3], "_")); fullpath <- paste0(basedir, "/", filename); read_delim(fullpath, delim=" ", col_names=c("Rank", "Start", "End", "Value"), progress=FALSE) %>% # Transform Value to factor mutate(Value = as.factor(Value)) %>% # Detect begin and end of iterations mutate(Iteration = case_when( (.$Value == "timste") ~ 1, (.$Value == "endste") ~ -1, TRUE ~ 0)) %>% group_by(Rank) %>% # Create a logical to detect observations within iterations mutate(Iteration = as.logical(cumsum(Iteration))) %>% # Get only observations that belongs to some iteration filter(Iteration == TRUE) %>% ungroup() %>% # Create the iteration by cumsum mutate(Iteration = case_when( (.$Value == "timste") ~ 1, TRUE ~ 0)) %>% group_by(Rank) %>% mutate(Iteration = cumsum(Iteration)) %>% mutate(EID = as.factor(meta[2]), Platform = as.factor(meta[3]), Nodes = as.factor(meta[4]), NP = as.factor(meta[5]), Partitioning = as.factor(meta[6]), Infiniband = as.logical(meta[7]), Alya = as.factor(meta[8])); } # Clean-up the files variable files <- strsplit(files, "\n")[[1]] # Do the reading, bind_rows everything dft16 <- do.call("bind_rows", lapply(files, function(file) { alya_scorep_trace_read(file, basedir=".") } ));#ata/07/17e0e2-99b8-44c7-acfc-4d4ad685a2c5") })); # Fix the columns factor dft16 <- dft16 %>% mutate(Partitioning = as.factor(Partitioning), Alya = as.factor(Alya)); dft16 %>% summary; #+end_src #+RESULTS: read_exp_16_traces #+begin_example Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Parsed with column specification: cols( Rank = col_integer(), Start = col_double(), End = col_double(), Value = col_character() ) Warning messages: 1: In bind_rows_(x, .id) : Unequal factor levels: coercing to character 2: In bind_rows_(x, .id) : Unequal factor levels: coercing to character Rank Start End Value Min. : 0.00 Min. : 0.243 Min. : 0.243 solver :1984000 1st Qu.: 31.75 1st Qu.: 629.751 1st Qu.: 629.751 nsi_updunk:1433600 Median : 63.50 Median :1206.153 Median :1206.207 nsi_solsgs: 793600 Mean : 63.50 Mean :1208.624 Mean :1208.801 nsi_updbcs: 652800 3rd Qu.: 95.25 3rd Qu.:1772.050 3rd Qu.:1772.199 nsi_inisol: 524800 Max. :127.00 Max. :2529.016 Max. :2529.016 nastin : 512000 (Other) :1958400 Iteration EID Platform Nodes NP Min. : 1.00 16-v1:7859200 grimoire:7859200 8:7859200 128:7859200 1st Qu.: 47.00 Median : 98.00 Mean : 98.34 3rd Qu.:149.00 Max. :200.00 Partitioning Infiniband Alya metis:4715520 Mode :logical Alya.x.modif:3143680 sfc :3143680 FALSE:1571840 Alya.x.orig :4715520 TRUE :6287360 NA's :0 #+end_example **** Filter to get only the computation states we are interested in #+begin_src R :results output :session :exports both dft16f <- dft16 %>% filter(Value %in% c("nsi_matrix", "solver")); dft16f #+end_src #+RESULTS: #+begin_example Source: local data frame [2,380,800 x 12] Groups: Rank [128] Rank Start End Value Iteration EID Platform Nodes NP 1 0 0.429466 0.429481 nsi_matrix 1 16-v1 grimoire 8 128 2 75 0.430559 1.132667 nsi_matrix 1 16-v1 grimoire 8 128 3 113 0.430731 1.190444 nsi_matrix 1 16-v1 grimoire 8 128 4 112 0.430729 1.204946 nsi_matrix 1 16-v1 grimoire 8 128 5 94 0.430749 1.206544 nsi_matrix 1 16-v1 grimoire 8 128 6 120 0.430872 1.208798 nsi_matrix 1 16-v1 grimoire 8 128 7 117 0.430724 1.210292 nsi_matrix 1 16-v1 grimoire 8 128 8 127 0.430743 1.210979 nsi_matrix 1 16-v1 grimoire 8 128 9 118 0.430760 1.217279 nsi_matrix 1 16-v1 grimoire 8 128 10 90 0.430767 1.219594 nsi_matrix 1 16-v1 grimoire 8 128 # ... with 2,380,790 more rows, and 3 more variables: Partitioning , # Infiniband , Alya #+end_example *** 2 Per-rank number of elements, points, etc (process and read) The files are obtained with manual instrumentation of Alya code. **** Process log files #+begin_src shell :results output EDIR=exp_16-v1_grimoire_8 for file in $(find $EDIR | grep results | grep log$); do OUTPUT=$(dirname $file)/$(basename $file .log).csv cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f1,3,5,7,9,11,13 | uniq > $OUTPUT cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f2,4,6,8,10,12,14 >> $OUTPUT head $OUTPUT tail $OUTPUT echo done #+end_src #+RESULTS: #+begin_example Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,22405,85371,172276,3477,4,22405 2,22923,86049,179769,3742,5,22923 3,24316,99541,175416,3029,5,24316 4,22907,90905,170125,3170,4,22907 5,22636,86964,173194,3448,5,22636 6,22774,88416,171711,3333,6,22774 7,23564,91625,177865,3452,7,23564 8,22211,83394,173364,3612,5,22211 9,23427,89980,179685,3588,4,23427 118,19344,51553,177808,5044,6,19344 119,19050,54006,174756,4829,3,19050 120,18742,53558,169698,4652,6,18742 121,19158,51974,179487,5103,4,19158 122,19820,53397,179497,5028,8,19820 123,19357,55432,172417,4679,6,19357 124,18855,51717,176727,5006,3,18855 125,19473,52093,179803,5107,6,19473 126,18948,49074,179769,5225,4,18948 127,19037,50759,172104,4863,6,19037 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,22405,85371,172276,3477,4,22405 2,22923,86049,179769,3742,5,22923 3,24316,99541,175416,3029,5,24316 4,22907,90905,170125,3170,4,22907 5,22636,86964,173194,3448,5,22636 6,22774,88416,171711,3333,6,22774 7,23564,91625,177865,3452,7,23564 8,22211,83394,173364,3612,5,22211 9,23427,89980,179685,3588,4,23427 118,19344,51553,177808,5044,6,19344 119,19050,54006,174756,4829,3,19050 120,18742,53558,169698,4652,6,18742 121,19158,51974,179487,5103,4,19158 122,19820,53397,179497,5028,8,19820 123,19357,55432,172417,4679,6,19357 124,18855,51717,176727,5006,3,18855 125,19473,52093,179803,5107,6,19473 126,18948,49074,179769,5225,4,18948 127,19037,50759,172104,4863,6,19037 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,14976,72536,72796,2384,17,14976 2,14641,72392,72575,448,12,14641 3,22264,72647,167221,4446,16,22264 4,28129,72464,251684,7187,11,28129 5,23239,72488,178813,4341,25,23239 6,26093,72525,209615,5640,14,26093 7,22996,72391,183401,4422,10,22996 8,21190,72393,160643,3505,7,21190 9,25178,72612,201462,5195,10,25178 118,24285,72483,188018,6181,14,24285 119,26512,72604,229879,6447,15,26512 120,24865,72374,200264,5134,10,24865 121,22912,72446,173477,4266,15,22912 122,14466,72460,73072,776,11,14466 123,14046,72589,72589,944,8,14046 124,15427,72509,73961,2155,15,15427 125,21776,72610,153930,4907,13,21776 126,21662,72595,161294,4243,12,21662 127,18484,72603,111590,4500,17,18484 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,22405,85371,172276,3477,4,22405 2,22923,86049,179769,3742,5,22923 3,24316,99541,175416,3029,5,24316 4,22907,90905,170125,3170,4,22907 5,22636,86964,173194,3448,5,22636 6,22774,88416,171711,3333,6,22774 7,23564,91625,177865,3452,7,23564 8,22211,83394,173364,3612,5,22211 9,23427,89980,179685,3588,4,23427 118,19344,51553,177808,5044,6,19344 119,19050,54006,174756,4829,3,19050 120,18742,53558,169698,4652,6,18742 121,19158,51974,179487,5103,4,19158 122,19820,53397,179497,5028,8,19820 123,19357,55432,172417,4679,6,19357 124,18855,51717,176727,5006,3,18855 125,19473,52093,179803,5107,6,19473 126,18948,49074,179769,5225,4,18948 127,19037,50759,172104,4863,6,19037 Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,32182,174168,174611,2973,16,32182 2,20919,53529,174758,5381,15,20919 3,19727,49972,175172,5020,11,19727 4,21918,69049,174054,4277,23,21918 5,21683,64345,174280,4439,14,21683 6,22205,64122,175002,4582,18,22205 7,21361,63432,174587,4436,11,21361 8,22162,75699,174459,3914,8,22162 9,21389,66339,174799,4332,8,21389 118,19736,47749,174249,5084,13,19736 119,27122,103080,174535,7852,12,27122 120,21785,59025,175145,5304,13,21785 121,20722,56163,175018,4869,12,20722 122,21269,59300,174615,4657,10,21269 123,22403,70519,174270,4135,13,22403 124,30100,145205,174682,2874,9,30100 125,31603,142203,174120,5271,15,31603 126,22843,73894,175386,4418,11,22843 127,26719,97390,174439,6359,13,26719 #+end_example **** Read in R #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_npoin <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[2], "_")); meta <- gsub(".dir", "", meta); read_csv(filename) %>% gather(Variable, Value, -Rank) %>% mutate(EID = meta[1], Platform = meta[2], Nodes = meta[3], NP = meta[4], Partitioning = meta[5], Infiniband = as.logical(meta[6]), Alya = meta[7]); } files <- list.files("exp_16-v1_grimoire_8", pattern="results_NPOIN_NELEM_NELEW_NBOUN.csv", recursive=TRUE, full.names=TRUE); dfp16 <- do.call("bind_rows", lapply(files, function(file) { read_npoin(file) })) dfp16 <- dfp16 %>% mutate(Partitioning=as.factor(Partitioning), NP=as.factor(NP), Alya=as.factor(Alya), Nodes=as.factor(Nodes), EID=as.factor(EID), Platform=as.factor(Platform)) dfp16 #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) # A tibble: 3,810 × 10 Rank Variable Value EID Platform Nodes NP Partitioning Infiniband 1 1 NPOIN 22405 16-v1 grimoire 8 128 metis FALSE 2 2 NPOIN 22923 16-v1 grimoire 8 128 metis FALSE 3 3 NPOIN 24316 16-v1 grimoire 8 128 metis FALSE 4 4 NPOIN 22907 16-v1 grimoire 8 128 metis FALSE 5 5 NPOIN 22636 16-v1 grimoire 8 128 metis FALSE 6 6 NPOIN 22774 16-v1 grimoire 8 128 metis FALSE 7 7 NPOIN 23564 16-v1 grimoire 8 128 metis FALSE 8 8 NPOIN 22211 16-v1 grimoire 8 128 metis FALSE 9 9 NPOIN 23427 16-v1 grimoire 8 128 metis FALSE 10 10 NPOIN 22871 16-v1 grimoire 8 128 metis FALSE # ... with 3,800 more rows, and 1 more variables: Alya #+end_example *** 3 Per-rank number of graph entries The data is obtained by recent changes in Alya, they are in the =partition.par.post.res= file. **** Process log files To process such files, I need to extract all entries in section identified by =NUMBER_NODE_GRAPH_ENTRIES=. To do so, #+begin_src shell :results output EDIR=exp_16-v1_grimoire_8 for file in $(find $EDIR | grep partition.par.post.res$); do OUTPUT=$(dirname $file)/$(basename $file .log)-entries.csv NP=$(echo $file | cut -d"/" -f2 | cut -d"_" -f4) START_LINE=$(cat -n $file | grep NUMBER_NODE_GRAPH_ENTRIES | grep ComponentNames | tr '\t' ' ' | sed "s/^[[:space:]]*//" | cut -d" " -f1) START_LINE=$(($START_LINE + 2)) END_LINE=$(($START_LINE + $NP - 1)) rm -f $OUTPUT echo "Rank,ENTRIES" >> $OUTPUT sed -n "${START_LINE},${END_LINE}p" $file | tr -s ' ' | sed "s/^ //" | tr ' ' ',' >> $OUTPUT head $OUTPUT tail $OUTPUT echo done #+end_src #+RESULTS: #+begin_example Rank,ENTRIES 1,0.360783E+06 2,0.369863E+06 3,0.381955E+06 4,0.389698E+06 5,0.371953E+06 6,0.372738E+06 7,0.372212E+06 8,0.385382E+06 9,0.369259E+06 119,0.348536E+06 120,0.342990E+06 121,0.335094E+06 122,0.348674E+06 123,0.354342E+06 124,0.343209E+06 125,0.343299E+06 126,0.351681E+06 127,0.346950E+06 128,0.339977E+06 Rank,ENTRIES 1,0.360783E+06 2,0.369863E+06 3,0.381955E+06 4,0.389698E+06 5,0.371953E+06 6,0.372738E+06 7,0.372212E+06 8,0.385382E+06 9,0.369259E+06 119,0.348536E+06 120,0.342990E+06 121,0.335094E+06 122,0.348674E+06 123,0.354342E+06 124,0.343209E+06 125,0.343299E+06 126,0.351681E+06 127,0.346950E+06 128,0.339977E+06 Rank,ENTRIES 1,0.372646E+06 2,0.201132E+06 3,0.198459E+06 4,0.360684E+06 5,0.499009E+06 6,0.381273E+06 7,0.436721E+06 8,0.384670E+06 9,0.346130E+06 119,0.399209E+06 120,0.462622E+06 121,0.417481E+06 122,0.372852E+06 123,0.197842E+06 124,0.194630E+06 125,0.205185E+06 126,0.342284E+06 127,0.349014E+06 128,0.269424E+06 Rank,ENTRIES 1,0.675840E+05 2,0.733730E+05 3,0.675900E+05 4,0.679910E+05 5,0.792140E+05 6,0.687050E+05 7,0.672740E+05 8,0.781860E+05 9,0.697650E+05 119,0.665170E+05 120,0.660530E+05 121,0.680130E+05 122,0.693500E+05 123,0.704970E+05 124,0.677320E+05 125,0.659370E+05 126,0.718380E+05 127,0.664600E+05 128,0.671620E+05 Rank,ENTRIES 1,0.371651E+06 2,0.456808E+06 3,0.356423E+06 4,0.348377E+06 5,0.365696E+06 6,0.363469E+06 7,0.368031E+06 8,0.361453E+06 9,0.369350E+06 119,0.346784E+06 120,0.408946E+06 121,0.364677E+06 122,0.356768E+06 123,0.360449E+06 124,0.370175E+06 125,0.437280E+06 126,0.446339E+06 127,0.374283E+06 128,0.403337E+06 #+end_example **** Read in R #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_entries <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[2], "_")); meta <- gsub(".dir", "", meta); read_csv(filename) %>% gather(Variable, Value, -Rank) %>% mutate(EID = meta[1], Platform = meta[2], Nodes = meta[3], NP = meta[4], Partitioning = meta[5], Infiniband = as.logical(meta[6]), Alya = meta[7]); } files <- list.files("exp_16-v1_grimoire_8", pattern="fensap-partition.par.post.res-entries.csv", recursive=TRUE, full.names=TRUE); dfe16 <- do.call("bind_rows", lapply(files, function(file) { read_entries(file) })) dfe16 <- dfe16 %>% mutate(Partitioning=as.factor(Partitioning), NP=as.factor(NP), Alya=as.factor(Alya), Nodes=as.factor(Nodes), EID=as.factor(EID), Platform=as.factor(Platform)) %>% # Here ranks are identified starting from 1, so: # we must reduce their values by 1 to match # the trace identification mutate(Rank = Rank - 1) dfe16 dfe16 %>% summary #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Rank = col_integer(), ENTRIES = col_double() ) Parsed with column specification: cols( Rank = col_integer(), ENTRIES = col_double() ) Parsed with column specification: cols( Rank = col_integer(), ENTRIES = col_double() ) Parsed with column specification: cols( Rank = col_integer(), ENTRIES = col_double() ) Parsed with column specification: cols( Rank = col_integer(), ENTRIES = col_double() ) # A tibble: 640 × 10 Rank Variable Value EID Platform Nodes NP Partitioning Infiniband 1 0 ENTRIES 360783 16-v1 grimoire 8 128 metis FALSE 2 1 ENTRIES 369863 16-v1 grimoire 8 128 metis FALSE 3 2 ENTRIES 381955 16-v1 grimoire 8 128 metis FALSE 4 3 ENTRIES 389698 16-v1 grimoire 8 128 metis FALSE 5 4 ENTRIES 371953 16-v1 grimoire 8 128 metis FALSE 6 5 ENTRIES 372738 16-v1 grimoire 8 128 metis FALSE 7 6 ENTRIES 372212 16-v1 grimoire 8 128 metis FALSE 8 7 ENTRIES 385382 16-v1 grimoire 8 128 metis FALSE 9 8 ENTRIES 369259 16-v1 grimoire 8 128 metis FALSE 10 9 ENTRIES 386269 16-v1 grimoire 8 128 metis FALSE # ... with 630 more rows, and 1 more variables: Alya Rank Variable Value EID Min. : 0.00 Length:640 Min. : 60096 16-v1:640 1st Qu.: 31.75 Class :character 1st Qu.:311744 Median : 63.50 Mode :character Median :355139 Mean : 63.50 Mean :306673 3rd Qu.: 95.25 3rd Qu.:374714 Max. :127.00 Max. :524192 Platform Nodes NP Partitioning Infiniband grimoire:640 8:640 128:640 metis:384 Mode :logical sfc :256 FALSE:128 TRUE :512 NA's :0 Alya Alya.x.modif:256 Alya.x.orig :384 #+end_example *** Merge trace and per-rank non-temporal data #+begin_src R :results output :session :exports both library(tidyr); dft16z <- dft16f %>% # Get all ranks except rank 0 filter(Rank != 0) %>% # Create a new variable named by the concation of the value (kernel name) and iteration mutate(Variable = Value) %>% ##paste0(Value, "-", Iteration)) %>% # Calculate the duration (put in the Value column) & remove unnecessary columns mutate(Value = End - Start) %>% select(-Start, -End) %>% select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% # Since we have several calls to our kernels for each iteration, we need to sum up group_by(Rank, Variable, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya) %>% summarize(N=n(), Value=sum(Value)) %>% ungroup() %>% # Impose an order select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya); dfp16z <- dfp16 %>% mutate(Iteration = NA) %>% select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya); dfe16z <- dfe16 %>% mutate(Iteration = NA) %>% select(Rank, Variable, Value, Iteration, EID, Platform, Nodes, NP, Partitioning, Infiniband, Alya); dfm16 <- bind_rows(dft16z, dfp16z, dfe16z); dfm16 <- dfm16 %>% mutate(Iteration = as.factor(Iteration), NP = factor(NP), Partitioning = factor(Partitioning), Alya = factor(Alya), Variable = factor(Variable)); dfm16 %>% summary dfm16 %>% .$Variable %>% unique #+end_src #+RESULTS: #+begin_example Warning message: In bind_rows_(x, .id) : binding factor and character vector, coercing into character vector Rank Variable Value Iteration Min. : 0 nsi_matrix:127000 Min. : 1.0 1 : 1270 1st Qu.: 32 solver :127000 1st Qu.: 2.7 2 : 1270 Median : 64 ENTRIES : 640 Median : 4.0 3 : 1270 Mean : 64 NBBOU : 635 Mean : 1488.2 4 : 1270 3rd Qu.: 96 NBOUN : 635 3rd Qu.: 5.8 5 : 1270 Max. :127 NELEM : 635 Max. :524192.0 (Other):247650 (Other) : 1905 NA's : 4450 EID Platform Nodes NP Partitioning 16-v1:258450 grimoire:258450 8:258450 128:258450 metis:155070 sfc :103380 Infiniband Alya Mode :logical Alya.x.modif:103380 FALSE:51690 Alya.x.orig :155070 TRUE :206760 NA's :0 [1] nsi_matrix solver NPOIN NELEM NELEW NBOUN NNEIG [8] NBBOU ENTRIES Levels: ENTRIES NBBOU NBOUN NELEM NELEW NNEIG NPOIN nsi_matrix solver #+end_example *** Plot the correlation between computaton time and NELEM, NELEW, ENTRIES #+begin_src R :results output graphics :file img/exp_16_solver_partition_entries.png :exports both :width 800 :height 600 :session library(cowplot); dfm16 %>% filter(Partitioning == "sfc") %>% filter(Infiniband == TRUE) %>% filter(is.na(Iteration) | Iteration %in% c(50, 100, 150, 200)) %>% filter((Variable %in% c("NELEM", "NELEW", "ENTRIES")) | grepl("solver", Variable)) -> dfm16sel; title = paste("Experiment:", (dfm16sel %>% .$EID %>% unique), (dfm16sel %>% .$Platform %>% unique), (dfm16sel %>% .$Nodes %>% unique), (dfm16sel %>% .$NP %>% unique), (dfm16sel %>% .$Partitioning %>% unique), (dfm16sel %>% .$Infiniband %>% unique), sep = " "); dfm16sel %>% ggplot(aes(x=Rank, y=Value, color=Iteration)) + theme_bw(base_size=14) + geom_point() + ylim(0,NA) + theme(legend.position="top", legend.direction="horizontal", panel.margin = unit(.08, "lines"), legend.spacing = unit(.0, "lines"), legend.margin=margin(b = -.3, unit='cm'), plot.margin = unit(x = c(0, 0, 0, 0), units = "mm")) + ggtitle (title) + facet_grid(Variable~Alya, scales="free_y") -> p; save_plot("img/exp_16_solver_partition_entries.pdf", p, base_aspect_ratio = 3.5); p; #+end_src #+RESULTS: [[file:img/exp_16_solver_partition_entries.png]] *** Check doiter to verify iteration duration #+begin_src R :results output :session :exports both dft16 %>% group_by(Iteration, Partitioning, Infiniband, Alya, Platform, EID, Nodes, NP) %>% summarize(Duration = max(End)-min(Start)) %>% ungroup() %>% filter(Partitioning == "sfc", Infiniband == TRUE) %>% select(-Platform, -EID, -Nodes, -NP) %>% head(n=100) %>% as.data.frame() #+end_src #+RESULTS: #+begin_example Iteration Partitioning Infiniband Alya Duration 1 1 sfc TRUE Alya.x.modif 17.756578 2 1 sfc TRUE Alya.x.orig 20.941277 3 2 sfc TRUE Alya.x.modif 17.117745 4 2 sfc TRUE Alya.x.orig 20.327477 5 3 sfc TRUE Alya.x.modif 17.066452 6 3 sfc TRUE Alya.x.orig 20.412655 7 4 sfc TRUE Alya.x.modif 16.960388 8 4 sfc TRUE Alya.x.orig 20.091386 9 5 sfc TRUE Alya.x.modif 16.866464 10 5 sfc TRUE Alya.x.orig 20.008922 11 6 sfc TRUE Alya.x.modif 16.846428 12 6 sfc TRUE Alya.x.orig 19.990011 13 7 sfc TRUE Alya.x.modif 16.496945 14 7 sfc TRUE Alya.x.orig 20.041291 15 8 sfc TRUE Alya.x.modif 16.504957 16 8 sfc TRUE Alya.x.orig 19.540478 17 9 sfc TRUE Alya.x.modif 16.239494 18 9 sfc TRUE Alya.x.orig 19.506054 19 10 sfc TRUE Alya.x.modif 13.295363 20 10 sfc TRUE Alya.x.orig 15.737029 21 11 sfc TRUE Alya.x.modif 13.309243 22 11 sfc TRUE Alya.x.orig 15.792624 23 12 sfc TRUE Alya.x.modif 10.215306 24 12 sfc TRUE Alya.x.orig 12.058212 25 13 sfc TRUE Alya.x.modif 9.988408 26 13 sfc TRUE Alya.x.orig 12.180781 27 14 sfc TRUE Alya.x.modif 10.165395 28 14 sfc TRUE Alya.x.orig 12.023906 29 15 sfc TRUE Alya.x.modif 10.110094 30 15 sfc TRUE Alya.x.orig 11.953535 31 16 sfc TRUE Alya.x.modif 10.148953 32 16 sfc TRUE Alya.x.orig 11.879859 33 17 sfc TRUE Alya.x.modif 10.057250 34 17 sfc TRUE Alya.x.orig 12.007429 35 18 sfc TRUE Alya.x.modif 10.186872 36 18 sfc TRUE Alya.x.orig 11.914862 37 19 sfc TRUE Alya.x.modif 10.128210 38 19 sfc TRUE Alya.x.orig 11.862125 39 20 sfc TRUE Alya.x.modif 9.999596 40 20 sfc TRUE Alya.x.orig 11.919397 41 21 sfc TRUE Alya.x.modif 10.091912 42 21 sfc TRUE Alya.x.orig 12.247608 43 22 sfc TRUE Alya.x.modif 10.104880 44 22 sfc TRUE Alya.x.orig 12.039911 45 23 sfc TRUE Alya.x.modif 9.919457 46 23 sfc TRUE Alya.x.orig 11.934311 47 24 sfc TRUE Alya.x.modif 9.991665 48 24 sfc TRUE Alya.x.orig 11.993532 49 25 sfc TRUE Alya.x.modif 10.103905 50 25 sfc TRUE Alya.x.orig 11.880654 51 26 sfc TRUE Alya.x.modif 10.190846 52 26 sfc TRUE Alya.x.orig 11.911790 53 27 sfc TRUE Alya.x.modif 10.105440 54 27 sfc TRUE Alya.x.orig 11.895977 55 28 sfc TRUE Alya.x.modif 10.087383 56 28 sfc TRUE Alya.x.orig 12.017077 57 29 sfc TRUE Alya.x.modif 9.849281 58 29 sfc TRUE Alya.x.orig 11.878089 59 30 sfc TRUE Alya.x.modif 9.890735 60 30 sfc TRUE Alya.x.orig 11.919046 61 31 sfc TRUE Alya.x.modif 10.014491 62 31 sfc TRUE Alya.x.orig 11.833787 63 32 sfc TRUE Alya.x.modif 9.874768 64 32 sfc TRUE Alya.x.orig 11.929741 65 33 sfc TRUE Alya.x.modif 9.846555 66 33 sfc TRUE Alya.x.orig 11.786520 67 34 sfc TRUE Alya.x.modif 9.994509 68 34 sfc TRUE Alya.x.orig 11.853944 69 35 sfc TRUE Alya.x.modif 9.700519 70 35 sfc TRUE Alya.x.orig 11.885302 71 36 sfc TRUE Alya.x.modif 9.728538 72 36 sfc TRUE Alya.x.orig 11.809225 73 37 sfc TRUE Alya.x.modif 9.683195 74 37 sfc TRUE Alya.x.orig 11.887436 75 38 sfc TRUE Alya.x.modif 9.729218 76 38 sfc TRUE Alya.x.orig 11.993540 77 39 sfc TRUE Alya.x.modif 9.638608 78 39 sfc TRUE Alya.x.orig 11.743263 79 40 sfc TRUE Alya.x.modif 9.652496 80 40 sfc TRUE Alya.x.orig 11.822727 81 41 sfc TRUE Alya.x.modif 9.731062 82 41 sfc TRUE Alya.x.orig 11.695359 83 42 sfc TRUE Alya.x.modif 9.711456 84 42 sfc TRUE Alya.x.orig 11.847502 85 43 sfc TRUE Alya.x.modif 9.571125 86 43 sfc TRUE Alya.x.orig 11.761173 87 44 sfc TRUE Alya.x.modif 9.561946 88 44 sfc TRUE Alya.x.orig 11.770252 89 45 sfc TRUE Alya.x.modif 9.715087 90 45 sfc TRUE Alya.x.orig 11.695930 91 46 sfc TRUE Alya.x.modif 9.623916 92 46 sfc TRUE Alya.x.orig 11.655520 93 47 sfc TRUE Alya.x.modif 9.627460 94 47 sfc TRUE Alya.x.orig 11.747964 95 48 sfc TRUE Alya.x.modif 9.634499 96 48 sfc TRUE Alya.x.orig 11.622833 97 49 sfc TRUE Alya.x.modif 9.584689 98 49 sfc TRUE Alya.x.orig 11.777986 99 50 sfc TRUE Alya.x.modif 9.727763 100 50 sfc TRUE Alya.x.orig 11.625774 #+end_example *** Sum all functions that have been traced to get compute time #+begin_src R :results output :session :exports both dft16 %>% filter(Partitioning == "sfc", Infiniband == TRUE) %>% mutate(Duration = End-Start) %>% group_by(Iteration, Rank, Value, Partitioning, Infiniband, Alya, Platform, EID, Nodes, NP) %>% summarize(N=n(), D=sum(Duration)) %>% filter(Iteration == 50) -> ret; ret; #+end_src #+RESULTS: #+begin_example # A tibble: 3,840 x 12 # Groups: Iteration, Rank, Value, Partitioning, Infiniband, Alya, Platform, # EID, Nodes [3,840] Iteration Rank Value Partitioning Infiniband Alya Platform 1 50 0 doiter sfc TRUE Alya.x.modif grimoire 2 50 0 doiter sfc TRUE Alya.x.orig grimoire 3 50 0 nastin sfc TRUE Alya.x.modif grimoire 4 50 0 nastin sfc TRUE Alya.x.orig grimoire 5 50 0 nsi_begste sfc TRUE Alya.x.modif grimoire 6 50 0 nsi_begste sfc TRUE Alya.x.orig grimoire 7 50 0 nsi_concou sfc TRUE Alya.x.modif grimoire 8 50 0 nsi_concou sfc TRUE Alya.x.orig grimoire 9 50 0 nsi_doiter sfc TRUE Alya.x.modif grimoire 10 50 0 nsi_doiter sfc TRUE Alya.x.orig grimoire # ... with 3,830 more rows, and 5 more variables: EID , Nodes , # NP , N , D #+end_example #+begin_src R :results output :session :exports both ret %>% ungroup() %>% filter(grepl("modif", Alya)) %>% head(n=100) %>% select(-Platform, -EID, -Nodes, -NP) %>% as.data.frame() #+end_src #+RESULTS: #+begin_example Iteration Rank Value Partitioning Infiniband Alya N D 1 50 0 doiter sfc TRUE Alya.x.modif 1 0.000010 2 50 0 nastin sfc TRUE Alya.x.modif 4 0.000002 3 50 0 nsi_begste sfc TRUE Alya.x.modif 1 0.000002 4 50 0 nsi_concou sfc TRUE Alya.x.modif 1 0.000009 5 50 0 nsi_doiter sfc TRUE Alya.x.modif 1 0.000001 6 50 0 nsi_endste sfc TRUE Alya.x.modif 1 0.000001 7 50 0 nsi_ifconf sfc TRUE Alya.x.modif 3 0.000002 8 50 0 nsi_inisol sfc TRUE Alya.x.modif 4 0.000009 9 50 0 nsi_matrix sfc TRUE Alya.x.modif 3 0.000045 10 50 0 nsi_solite sfc TRUE Alya.x.modif 3 0.000685 11 50 0 nsi_solsgs sfc TRUE Alya.x.modif 6 1.519370 12 50 0 nsi_updbcs sfc TRUE Alya.x.modif 5 0.000008 13 50 0 nsi_updunk sfc TRUE Alya.x.modif 11 0.000010 14 50 0 solver sfc TRUE Alya.x.modif 15 8.010709 15 50 0 timste sfc TRUE Alya.x.modif 1 0.000242 16 50 1 doiter sfc TRUE Alya.x.modif 1 0.000010 17 50 1 nastin sfc TRUE Alya.x.modif 4 0.000001 18 50 1 nsi_begste sfc TRUE Alya.x.modif 1 0.000002 19 50 1 nsi_concou sfc TRUE Alya.x.modif 1 0.000001 20 50 1 nsi_doiter sfc TRUE Alya.x.modif 1 0.000001 21 50 1 nsi_endste sfc TRUE Alya.x.modif 1 0.000435 22 50 1 nsi_ifconf sfc TRUE Alya.x.modif 3 0.000001 23 50 1 nsi_inisol sfc TRUE Alya.x.modif 4 0.000018 24 50 1 nsi_matrix sfc TRUE Alya.x.modif 3 1.896372 25 50 1 nsi_solite sfc TRUE Alya.x.modif 3 0.000632 26 50 1 nsi_solsgs sfc TRUE Alya.x.modif 6 1.518940 27 50 1 nsi_updbcs sfc TRUE Alya.x.modif 5 0.000282 28 50 1 nsi_updunk sfc TRUE Alya.x.modif 11 0.004265 29 50 1 solver sfc TRUE Alya.x.modif 15 5.649254 30 50 1 timste sfc TRUE Alya.x.modif 1 0.000006 31 50 2 doiter sfc TRUE Alya.x.modif 1 0.000008 32 50 2 nastin sfc TRUE Alya.x.modif 4 0.000004 33 50 2 nsi_begste sfc TRUE Alya.x.modif 1 0.000001 34 50 2 nsi_concou sfc TRUE Alya.x.modif 1 0.000001 35 50 2 nsi_doiter sfc TRUE Alya.x.modif 1 0.000000 36 50 2 nsi_endste sfc TRUE Alya.x.modif 1 0.000369 37 50 2 nsi_ifconf sfc TRUE Alya.x.modif 3 0.000001 38 50 2 nsi_inisol sfc TRUE Alya.x.modif 4 0.000011 39 50 2 nsi_matrix sfc TRUE Alya.x.modif 3 1.868698 40 50 2 nsi_solite sfc TRUE Alya.x.modif 3 0.000310 41 50 2 nsi_solsgs sfc TRUE Alya.x.modif 6 1.518933 42 50 2 nsi_updbcs sfc TRUE Alya.x.modif 5 0.000163 43 50 2 nsi_updunk sfc TRUE Alya.x.modif 11 0.003600 44 50 2 solver sfc TRUE Alya.x.modif 15 5.705557 45 50 2 timste sfc TRUE Alya.x.modif 1 0.000005 46 50 3 doiter sfc TRUE Alya.x.modif 1 0.000008 47 50 3 nastin sfc TRUE Alya.x.modif 4 0.000002 48 50 3 nsi_begste sfc TRUE Alya.x.modif 1 0.000001 49 50 3 nsi_concou sfc TRUE Alya.x.modif 1 0.000001 50 50 3 nsi_doiter sfc TRUE Alya.x.modif 1 0.000000 51 50 3 nsi_endste sfc TRUE Alya.x.modif 1 0.000498 52 50 3 nsi_ifconf sfc TRUE Alya.x.modif 3 0.000001 53 50 3 nsi_inisol sfc TRUE Alya.x.modif 4 0.000005 54 50 3 nsi_matrix sfc TRUE Alya.x.modif 3 2.669050 55 50 3 nsi_solite sfc TRUE Alya.x.modif 3 0.000575 56 50 3 nsi_solsgs sfc TRUE Alya.x.modif 6 1.518712 57 50 3 nsi_updbcs sfc TRUE Alya.x.modif 5 0.000274 58 50 3 nsi_updunk sfc TRUE Alya.x.modif 11 0.005965 59 50 3 solver sfc TRUE Alya.x.modif 15 4.819236 60 50 3 timste sfc TRUE Alya.x.modif 1 0.000006 61 50 4 doiter sfc TRUE Alya.x.modif 1 0.000009 62 50 4 nastin sfc TRUE Alya.x.modif 4 0.000000 63 50 4 nsi_begste sfc TRUE Alya.x.modif 1 0.000002 64 50 4 nsi_concou sfc TRUE Alya.x.modif 1 0.000001 65 50 4 nsi_doiter sfc TRUE Alya.x.modif 1 0.000001 66 50 4 nsi_endste sfc TRUE Alya.x.modif 1 0.000576 67 50 4 nsi_ifconf sfc TRUE Alya.x.modif 3 0.000002 68 50 4 nsi_inisol sfc TRUE Alya.x.modif 4 0.000007 69 50 4 nsi_matrix sfc TRUE Alya.x.modif 3 3.379627 70 50 4 nsi_solite sfc TRUE Alya.x.modif 3 0.000760 71 50 4 nsi_solsgs sfc TRUE Alya.x.modif 6 1.518519 72 50 4 nsi_updbcs sfc TRUE Alya.x.modif 5 0.000369 73 50 4 nsi_updunk sfc TRUE Alya.x.modif 11 0.006534 74 50 4 solver sfc TRUE Alya.x.modif 15 4.039487 75 50 4 timste sfc TRUE Alya.x.modif 1 0.000005 76 50 5 doiter sfc TRUE Alya.x.modif 1 0.000007 77 50 5 nastin sfc TRUE Alya.x.modif 4 0.000002 78 50 5 nsi_begste sfc TRUE Alya.x.modif 1 0.000002 79 50 5 nsi_concou sfc TRUE Alya.x.modif 1 0.000000 80 50 5 nsi_doiter sfc TRUE Alya.x.modif 1 0.000000 81 50 5 nsi_endste sfc TRUE Alya.x.modif 1 0.000514 82 50 5 nsi_ifconf sfc TRUE Alya.x.modif 3 0.000001 83 50 5 nsi_inisol sfc TRUE Alya.x.modif 4 0.000008 84 50 5 nsi_matrix sfc TRUE Alya.x.modif 3 2.733239 85 50 5 nsi_solite sfc TRUE Alya.x.modif 3 0.000602 86 50 5 nsi_solsgs sfc TRUE Alya.x.modif 6 1.518674 87 50 5 nsi_updbcs sfc TRUE Alya.x.modif 5 0.000279 88 50 5 nsi_updunk sfc TRUE Alya.x.modif 11 0.006134 89 50 5 solver sfc TRUE Alya.x.modif 15 4.745333 90 50 5 timste sfc TRUE Alya.x.modif 1 0.000005 91 50 6 doiter sfc TRUE Alya.x.modif 1 0.000008 92 50 6 nastin sfc TRUE Alya.x.modif 4 0.000003 93 50 6 nsi_begste sfc TRUE Alya.x.modif 1 0.000001 94 50 6 nsi_concou sfc TRUE Alya.x.modif 1 0.000001 95 50 6 nsi_doiter sfc TRUE Alya.x.modif 1 0.000000 96 50 6 nsi_endste sfc TRUE Alya.x.modif 1 0.000550 97 50 6 nsi_ifconf sfc TRUE Alya.x.modif 3 0.000001 98 50 6 nsi_inisol sfc TRUE Alya.x.modif 4 0.000007 99 50 6 nsi_matrix sfc TRUE Alya.x.modif 3 3.059509 100 50 6 nsi_solite sfc TRUE Alya.x.modif 3 0.000682 #+end_example ** 15-node chetemi :EXP17: Goal: Check Score-P profile with per-region MPI counts *** Alya.f90 Instrumentation #+begin_src shell :results output cd ~/misc/alya-bsc/ svn diff Sources/kernel/master/Alya.f90 #+end_src #+RESULTS: #+begin_example Index: Sources/kernel/master/Alya.f90 =================================================================== --- Sources/kernel/master/Alya.f90 (revision 8414) +++ Sources/kernel/master/Alya.f90 (working copy) @@ -1,3 +1,4 @@ +#include "scorep/SCOREP_User.inc" !> @file Alya.f90 !! @author Guillaume Houzeaux !! @brief Ayla main @@ -21,6 +22,9 @@ use def_master, only : kfl_gocou use def_coupli, only : kfl_gozon implicit none + INTEGER :: iter + character*100 striter + SCOREP_USER_REGION_DEFINE(lucas) ! ! DLB should be disabled as we only wabnt to activate it for particular loops ! Master does not disble to lend its resources automatically @@ -28,7 +32,7 @@ #ifdef ALYA_DLB include 'dlbf.h' if( INOTMASTER ) then - if( dlb_disable() < DLB_SUCCESS ) call par_livinf(17_ip,'DLB COULD NOT BE DISABLED',0_ip) + if( dlb_disable() < DLB_SUCCESS ) call runend('ALYA: DLB DOES NOT WORK PROPERLY') end if #endif #ifdef EXTRAE @@ -39,6 +43,7 @@ call Parall(22270_ip) + iter = 1 optimization: do while ( kfl_goopt == 1 ) call Iniunk() @@ -50,6 +55,9 @@ call Timste() + write(striter, '(a,i1)') 'iter',iter + SCOREP_USER_REGION_BY_NAME_BEGIN(striter, SCOREP_USER_REGION_TYPE_PHASE) + reset: do call Begste() @@ -77,6 +85,10 @@ call Endste() + SCOREP_USER_REGION_BY_NAME_END(striter) + iter = iter + 1 + lucas = SCOREP_USER_INVALID_REGION + call Filter(ITASK_ENDTIM) call Output(ITASK_ENDTIM) #+end_example *** Contents of the experiment #+begin_src shell :results output ls -lhR exp_17_chetemi_15_manual #+end_src #+RESULTS: #+begin_example exp_17_chetemi_15_manual: total 1.1M -rw-r----- 1 schnorr schnorr 227 Jan 23 06:44 fensap.ker.log -rw-r----- 1 schnorr schnorr 113K Jan 23 06:44 fensap.log -rw-r----- 1 schnorr schnorr 7.6K Jan 23 06:44 fensap.nsi.log -rw-r----- 1 schnorr schnorr 3.5K Jan 23 06:44 fensap.par.log -rw-r----- 1 schnorr schnorr 849K Jan 23 06:44 fensap-sgs.nsi.log -rw-r----- 1 schnorr schnorr 1.3K Jan 23 06:44 fensap-system.log -rw-r--r-- 1 schnorr schnorr 41K Jan 23 06:44 results_NPOIN_NELEM_NELEW_NBOUN.log drwxr-xr-x 2 schnorr schnorr 4.0K Jan 23 06:42 scorep-20180123_0156_642858649812104 exp_17_chetemi_15_manual/scorep-20180123_0156_642858649812104: total 1.4M -rw-r--r-- 1 schnorr schnorr 1.4M Jan 23 06:42 profile.cubex -rw-r--r-- 1 schnorr schnorr 1.3K Jan 23 06:42 scorep.cfg #+end_example *** Check profile.cubex You'll need a CUBE installation, with their auxiliary tools. #+name: exp17_cubex_to_open #+begin_src shell :results output cd exp_17_chetemi_15_manual/scorep-20180123_0156_642858649812104 ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp17_cubex_to_open *** Code from Arnaud to parse the Call tree #+name: exp17_cube_calltree #+header: :var dep0=exp17_cubex_to_open #+begin_src perl :results output :exports both use strict; my($filename) = "./exp_17_chetemi_15_manual/scorep-20180123_0156_642858649812104/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp17_cube_calltree : ./exp_17_chetemi_15_manual/scorep-20180123_0156_642858649812104/cube_info.csv *** Enrich the call tree with profile measurements #+name: exp17_enrich #+header: :var CSV=exp17_cube_calltree #+begin_src R :results output :session :exports both WD = "./exp_17_chetemi_15_manual/scorep-20180123_0156_642858649812104/"; PROFILE = paste0(WD, "profile.csv"); REGION = paste0(WD, "regions-codes.csv"); read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) -> d; df.PROF <- read_csv(PROFILE); df.REGION <- read_delim(REGION, col_names=FALSE, delim=";"); d %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) -> df #+end_src #+RESULTS: exp17_enrich #+begin_example Parsed with column specification: cols( X1 = col_integer(), X2 = col_character(), X3 = col_integer() ) Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 6594 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 4501 bytes_sent an integer 3570819600 file 2 4501 bytes_received an integer 3570819600 row 3 4502 bytes_sent an integer 3570819600 col 4 4502 bytes_received an integer 3570819600 expected 5 4503 bytes_sent an integer 3570819600 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Warning message: In rbind(names(probs), probs_f) : number of columns of result is not a multiple of vector length (arg 1) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) #+end_example *** 2 Per-rank number of elements, points, etc (process and read) **** Process log files #+name: exp17_log_to_csv_points #+begin_src shell :results output EDIR=exp_17_chetemi_15_manual/ for file in $(find $EDIR | grep results | grep log$); do OUTPUT=$(dirname $file)/$(basename $file .log).csv cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f1,3,5,7,9,11,13 | uniq > $OUTPUT cat $file | tr -s " " | sed "s/^ //" | tr ' ' ',' | cut -d"," -f2,4,6,8,10,12,14 >> $OUTPUT head $OUTPUT tail $OUTPUT echo done #+end_src #+RESULTS: exp17_log_to_csv_points #+begin_example Rank,NPOIN,NELEM,NELEW,NBOUN,NNEIG,NBBOU 1,15156,73772,74032,2384,18,15156 2,14967,74015,74198,471,13,14967 3,11537,43926,74370,1797,14,11537 4,8919,20973,73418,2127,13,8919 5,9211,20762,74212,2191,14,9211 6,8915,21347,74002,2097,9,8915 7,8657,20938,73503,2113,12,8657 8,9076,24188,73808,1979,10,9076 9,10327,34522,73757,1590,13,10327 290,9808,27318,74279,1877,15,9808 291,13682,63829,74331,1228,15,13682 292,14573,74413,74413,1097,8,14573 293,15587,72889,74341,1767,19,15587 294,13482,58001,74276,2750,15,13482 295,9462,25039,74549,1962,13,9462 296,9744,28896,74320,1986,10,9744 297,11034,37519,74875,1702,15,11034 298,10946,35249,73659,2651,12,10946 299,13374,51818,73849,3020,19,13374 #+end_example **** Read in R #+name: exp17_points #+header: :var dep0=exp17_log_to_csv_points #+begin_src R :results output :session :exports both library(readr); library(dplyr); library(tidyr); read_npoin <- function(filename) { meta <- unlist(strsplit(unlist(strsplit(filename, "/"))[2], "_")); meta <- gsub(".dir", "", meta); read_csv(filename); } files <- list.files("exp_17_chetemi_15_manual", pattern="results_NPOIN_NELEM_NELEW_NBOUN.csv", recursive=TRUE, full.names=TRUE); dfp17 <- do.call("bind_rows", lapply(files, function(file) { read_npoin(file) })) dfp17 #+end_src #+RESULTS: exp17_points #+begin_example Parsed with column specification: cols( Rank = col_integer(), NPOIN = col_integer(), NELEM = col_integer(), NELEW = col_integer(), NBOUN = col_integer(), NNEIG = col_integer(), NBBOU = col_integer() ) # A tibble: 299 x 7 Rank NPOIN NELEM NELEW NBOUN NNEIG NBBOU 1 1 15156 73772 74032 2384 18 15156 2 2 14967 74015 74198 471 13 14967 3 3 11537 43926 74370 1797 14 11537 4 4 8919 20973 73418 2127 13 8919 5 5 9211 20762 74212 2191 14 9211 6 6 8915 21347 74002 2097 9 8915 7 7 8657 20938 73503 2113 12 8657 8 8 9076 24188 73808 1979 10 9076 9 9 10327 34522 73757 1590 13 10327 10 10 9811 26661 73911 1962 26 9811 # ... with 289 more rows #+end_example *** Rough initial LB Analysis #+header: :var dep0=exp17_enrich #+header: :var dep1=exp17_points #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session df %>% left_join(dfp17) %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% filter(Rank != 0) %>% filter(Code == "Computation") %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.1) + ylim(0,NA) #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure24763VS2.png]] #+begin_src R :results output :session :exports both #+end_src *** Cumsum finally with plot (Hibert Load Curve) #+header: :var dep0=exp17_enrich #+header: :var dep1=exp17_points #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session m1=0 m2=m1+.05 NP = 299 df %>% left_join(dfp17) %>% filter(Rank != 0) %>% filter(Phase == 5) %>% filter(Code == "Computation") %>% arrange(Rank) %>% select(Rank, time, NELEW) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEWSUM = cumsum(NELEW)/sum(NELEW)) -> df.CUMSUM; df.CUMSUM %>% ggplot(aes(x=NELEWSUM, y=TIMESUM)) + geom_point() + geom_line() + ylim(m1,m2) + xlim(m1,m2) + geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure247630Oq.png]] *** Inverting Hilbert Load Curve #+name: exp17_hilbert #+begin_src R :results output :session *R* :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp17_hilbert #+header: :var dep0=exp17_hilbert #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEWSUM) -> df.CUMSUM.2 nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+RESULTS: #+begin_example # A tibble: 299 x 7 Rank time NELEW y x Target.x Target.y 1 1 26.696322 74032 0.006082734 0.003338458 0.001835591 0.003344482 2 2 26.975246 74198 0.012229020 0.006684402 0.003668480 0.006688963 3 3 18.678574 74370 0.016484917 0.010038101 0.005489164 0.010033445 4 4 12.662677 73418 0.019370097 0.013348871 0.007589754 0.013377926 5 5 6.610780 74212 0.020876357 0.016695446 0.010310625 0.016722408 6 6 12.895270 74002 0.023814533 0.020032551 0.014896989 0.020066890 7 7 12.642058 73503 0.026695015 0.023347154 0.019574650 0.023411371 8 8 8.923145 73808 0.028728146 0.026675511 0.023446748 0.026755853 9 9 16.234130 73757 0.032427079 0.030001567 0.027909374 0.030100334 10 10 14.480795 73911 0.035726515 0.033334569 0.031029658 0.033444816 # ... with 289 more rows #+end_example #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+0.02 NP = 299 df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=4, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) + geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-24763lPR/figure24763qjJ.png]] #+begin_src R :results output :session :exports both df_cum_inverse(df.CUMSUM.2, yval = 0.02) #+end_src #+RESULTS: : [1] 300 : [1] 0.02 : [1] 5 : x y : 5 0.01334887 0.01937010 : 6 0.01669545 0.02087636 : [1] 0.01474838 *** Provide Rick with a file with the number of elements The following is not necessary because Ricard is going to define a format and we will comply to it. #+begin_src R :results output :session :exports both dfp17 %>% select(Rank, NELEM) %>% write_csv("NELEM-300ranks.csv") #+end_src #+RESULTS: ** 50-node grisou + 8-node grimoire (928 cores + 1K iterations) :EXP18: *** Alya.f90 Instrumentation #+begin_src shell :results output cd ~/misc/alya-bsc/ svn diff Sources/kernel/master/Alya.f90 #+end_src #+RESULTS: #+begin_example Index: Sources/kernel/master/Alya.f90 =================================================================== --- Sources/kernel/master/Alya.f90 (revision 8439) +++ Sources/kernel/master/Alya.f90 (working copy) @@ -1,3 +1,4 @@ +#include "scorep/SCOREP_User.inc" !> @file Alya.f90 !! @author Guillaume Houzeaux !! @brief Ayla main @@ -20,7 +21,13 @@ use def_master, only : kfl_goblk use def_master, only : kfl_gocou use def_coupli, only : kfl_gozon + use mod_parall, only : PAR_MY_WORLD_RANK implicit none + INTEGER :: iter,ierror + character*100 striter + character*100 iterfile + real :: tnow + real, dimension(2) :: tarray ! ! DLB should be disabled as we only wabnt to activate it for particular loops ! Master does not disble to lend its resources automatically @@ -39,6 +46,10 @@ call Parall(22270_ip) + write(iterfile,'(a,i4.4,a)') 'iterations-', PAR_MY_WORLD_RANK, '.csv' + open(unit=2, file=iterfile) + + iter = 1 optimization: do while ( kfl_goopt == 1 ) call Iniunk() @@ -49,6 +60,11 @@ time: do while ( kfl_gotim == 1 ) call Timste() + call ETIME(tarray, tnow) + write(2,*) tnow, tarray(1), tarray(2), PAR_MY_WORLD_RANK, iter + + write(striter, '(a,i3.3)') 'iter',iter + SCOREP_USER_REGION_BY_NAME_BEGIN(striter, SCOREP_USER_REGION_TYPE_PHASE) reset: do call Begste() @@ -77,6 +93,13 @@ call Endste() + SCOREP_USER_REGION_BY_NAME_END(striter) + call ETIME(tarray, tnow) + write(2,*) tnow, tarray(1), tarray(2), PAR_MY_WORLD_RANK, iter + iter = iter + 1 + + + call Filter(ITASK_ENDTIM) call Output(ITASK_ENDTIM) @@ -91,6 +114,9 @@ end do optimization + close(2) + call Turnof() + end program Alya #+end_example *** config.in with scorep #+begin_src shell :results output cd ~/misc/alya-bsc/ head -n 25 Executables/unix/config.in #+end_src #+RESULTS: #+begin_example ################################################################### # GFORTRAN CONFIGURE # #MN3 RECOMENDED MODULE: # #module load gcc/5.1.0 openmpi/1.8.5 # ################################################################### #@Compiler: Using gfortran Compiler. SCOREP = ~/install/nova/scorep-3.0-alya/bin/scorep MPIF90 = ~//spack-ALYA/opt/spack/linux-debian9-x86_64/gcc-6.3.0/openmpi-3.0.0-a7g33v4ulwtb4g2verliyelvtifybrq3/bin/mpif90 MPICC = ~//spack-ALYA/opt/spack/linux-debian9-x86_64/gcc-6.3.0/openmpi-3.0.0-a7g33v4ulwtb4g2verliyelvtifybrq3/bin/mpicc F77 = $(SCOREP) --user --nocompiler --nopomp --noopenmp $(MPIF90) -cpp F90 = $(SCOREP) --user --nocompiler --nopomp --noopenmp $(MPIF90) -cpp FCOCC = $(SCOREP) --user --nocompiler --nopomp --noopenmp $(MPICC) -c FCFLAGS = -c -J$O -I$O -ffree-line-length-none -fimplicit-none FPPFLAGS = -x f95-cpp-input EXTRALIB = -lc EXTRAINC = fa2p = $(MPIF90) -cpp -c -x f95-cpp-input -DMPI_OFF -J../../Utils/user/alya2pos -I../../Utils/user/alya2pos fa2plk = $(MPIF90) -cpp -lc ################################################################### # PERFORMANCE FLAGS # ################################################################### #@Optimization: O1 FOPT = -O2 #+end_example *** Comments on tracing only COLL MPI group :deprecated: #+begin_src shell :results output export SCOREP_MPI_ENABLE_GROUPS=COLL export SCOREP_ENABLE_PROFILING=TRUE export SCOREP_ENABLE_TRACING=TRUE #+end_src If I do as above (only collectives), trace become big. And otf22csv is strange, it allocates all the memory before dumping anything. If I only enable the COLL group, profiling get affected by that and my per-phase compute mesure becomes unreliable. So, I've decided to write begin/end of each phase in files. *** Execution #+begin_src shell :results output source ~/spack-ALYA/share/spack/setup-env.sh export PATH=$(spack location -i openmpi)/bin:$PATH $(which mpirun) \ -x SCOREP_TOTAL_MEMORY=4GB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np $(cat ~/machine-file | wc -l) \ -machinefile ~/machine-file \ ~/alya-bsc/Executables/unix/Alya.x fensap #+end_src *** Details about each partition - Output the points (and details) for each partition - Carlos told me that I can put this to obtain some info #+BEGIN_EXAMPLE PARALL_SERVICE: On OUTPUT_FILE: On #+END_EXAMPLE *** 0. Experiment directory #+name: exp18_dir #+begin_src shell :results output echo -n "exp_18_grisou_grimoire_58_manual/np912_1000iter_scorep-20180125_0517_105434259013856/" #+end_src #+RESULTS: exp18_dir : exp_18_grisou_grimoire_58_manual/np912_1000iter_scorep-20180125_0517_105434259013856/ *** Data Transformation **** 1. Read Iteration Timings #+name: exp18_iteration_timings #+header: :var DIR=exp18_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src #+RESULTS: exp18_iteration_timings #+begin_example Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) #+end_example **** 2. Transform profile.cubex #+name: exp18_current_dir #+header: :var DIR=exp18_dir #+begin_src R :results verbatim :session :exports both print(DIR) #+end_src #+RESULTS: exp18_current_dir : exp_18_grisou_grimoire_58_manual/np912_1000iter_scorep-20180125_0517_105434259013856/ #+name: exp18_cubex_to_open #+header: :var DIR=exp18_current_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp18_cubex_to_open **** 3. Parse the call tree #+name: exp18_cube_calltree #+header: :var dep0=exp18_cubex_to_open #+header: :var DIR=exp18_current_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp18_cube_calltree : exp_18_grisou_grimoire_58_manual/np912_1000iter_scorep-20180125_0517_105434259013856//cube_info.csv **** 4. Enrich the call tree #+name: exp18_enrich #+header: :var CSV=exp18_cube_calltree #+header: :var DIR=exp18_current_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); CSV = "exp_18_grisou_grimoire_58_manual/np912_1000iter_scorep-20180125_0517_105434259013856//cube_info.csv" PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src **** X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+header: :var dep0=exp18_enrich #+begin_src R :results output graphics :file img/exp18_1K_iterations_912_NP.png :exports both :width 1400 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + #geom_line(aes(group=Rank), alpha=.1) + geom_point(alpha=.1, size=.1) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:img/exp18_1K_iterations_912_NP.png]] ***** Calculate the average #+name: exp18_lb_average #+header: :var dep0=exp18_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp18_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** 5. The number of elements on T1 #+name: exp18_number_of_elements_shell #+header: :var DIR=exp18_current_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+RESULTS: exp18_number_of_elements_shell | Type | Rank | NELEM | NPOIN | NBOUN | NPOI32 | | T1 | 0 | 0 | 0 | 553075 | -1 | | T1 | 1 | 143862 | 27303 | 2832 | 2362 | | T1 | 2 | 143921 | 48988 | 11415 | 4174 | | T1 | 3 | 143822 | 47644 | 9906 | 4947 | | T1 | 4 | 143695 | 43667 | 8197 | 3921 | | T1 | 5 | 143675 | 46242 | 9442 | 4191 | | T1 | 6 | 143752 | 43209 | 7433 | 4876 | | T1 | 7 | 143768 | 48927 | 11007 | 4580 | | T1 | 8 | 143792 | 47241 | 10828 | 2428 | | T1 | 9 | 143945 | 39022 | 6023 | 3238 | | T1 | 10 | 144243 | 47255 | 9907 | 4088 | | T1 | 11 | 143833 | 52178 | 12835 | 4720 | | T1 | 12 | 143842 | 48489 | 13095 | 2082 | | T1 | 13 | 143754 | 47833 | 10503 | 3770 | | T1 | 14 | 143817 | 35239 | 3680 | 3938 | | T1 | 15 | 143776 | 38633 | 5338 | 4204 | | T1 | 16 | 144023 | 37650 | 5066 | 3630 | | T1 | 17 | 143773 | 42217 | 7839 | 3269 | | T1 | 18 | 143746 | 32849 | 2614 | 3662 | | T1 | 19 | 143824 | 37433 | 4732 | 4118 | | T1 | 20 | 143821 | 38632 | 5583 | 3796 | | T1 | 21 | 143849 | 42960 | 8158 | 3320 | | T1 | 22 | 143789 | 46616 | 9577 | 4177 | | T1 | 23 | 143887 | 43733 | 8093 | 3975 | | T1 | 24 | 143817 | 40890 | 6690 | 3904 | | T1 | 25 | 143742 | 37534 | 4465 | 4698 | | T1 | 26 | 143741 | 34257 | 3128 | 3959 | | T1 | 27 | 143887 | 39461 | 6142 | 3517 | | T1 | 28 | 143876 | 35937 | 4111 | 3778 | | T1 | 29 | 144122 | 46780 | 9805 | 3777 | | T1 | 30 | 143919 | 47984 | 11611 | 2013 | | T1 | 31 | 143703 | 43909 | 9221 | 3191 | | T1 | 32 | 143864 | 41551 | 7138 | 3673 | | T1 | 33 | 143920 | 45680 | 9560 | 3251 | | T1 | 34 | 143917 | 43972 | 8473 | 3576 | | T1 | 35 | 143733 | 36957 | 5195 | 2808 | | T1 | 36 | 143974 | 37075 | 5274 | 2810 | | T1 | 37 | 143763 | 33188 | 2855 | 3421 | | T1 | 38 | 143915 | 43860 | 8409 | 3526 | | T1 | 39 | 143733 | 43199 | 7629 | 4442 | | T1 | 40 | 144072 | 50989 | 12285 | 3312 | | T1 | 41 | 143873 | 34536 | 7704 | 3727 | | T1 | 42 | 143719 | 36628 | 6242 | 3728 | | T1 | 43 | 143858 | 46666 | 11830 | 3627 | | T1 | 44 | 143779 | 49048 | 12550 | 4756 | | T1 | 45 | 143927 | 50855 | 11622 | 4467 | | T1 | 46 | 143677 | 51004 | 11178 | 4918 | | T1 | 47 | 143977 | 51247 | 11323 | 4801 | | T1 | 48 | 143872 | 49662 | 11307 | 3910 | | T1 | 49 | 144004 | 43839 | 8154 | 3973 | | T1 | 50 | 143728 | 43759 | 8449 | 3406 | | T1 | 51 | 143841 | 41709 | 7511 | 3472 | | T1 | 52 | 143929 | 50158 | 11469 | 4245 | | T1 | 53 | 143971 | 41111 | 6898 | 3765 | | T1 | 54 | 143831 | 55000 | 13607 | 4988 | | T1 | 55 | 143580 | 55935 | 14390 | 4804 | | T1 | 56 | 143751 | 56151 | 13884 | 6112 | | T1 | 57 | 143993 | 53463 | 13028 | 4431 | | T1 | 58 | 144026 | 55324 | 14121 | 4205 | | T1 | 59 | 143762 | 46085 | 12560 | 4506 | | T1 | 60 | 143832 | 50092 | 12743 | 4424 | | T1 | 61 | 143674 | 44626 | 8912 | 3867 | | T1 | 62 | 143894 | 26865 | 1694 | 2141 | | T1 | 63 | 143735 | 36132 | 7190 | 4182 | | T1 | 64 | 144024 | 38547 | 8615 | 3754 | #+name: exp18_number_of_elements #+header: :var TABLE=exp18_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp18_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp18_integrate #+header: :var dep1=exp18_lb_average #+header: :var dep0=exp18_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp18_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 28.18224 143862 2 2 41.61312 143921 3 3 40.45403 143822 4 4 38.41553 143695 5 5 40.30081 143675 6 6 37.42849 143752 7 7 42.15025 143768 8 8 42.18351 143792 9 9 36.24237 143945 10 10 40.10288 144243 # ... with 54 more rows #+end_example **** 7. Do the cumsum #+name: exp18_cumsum #+header: :var dep0=exp18_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp18.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp18_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 60.43743 184387 0.01510068 0.02002865 2 2 68.98525 135390 0.03233709 0.03473510 3 3 62.07687 134635 0.04784740 0.04935954 4 4 61.41787 140751 0.06319305 0.06464832 5 5 66.77389 137213 0.07987694 0.07955279 6 6 60.27102 146712 0.09493604 0.09548907 7 7 69.88611 141375 0.11239754 0.11084562 8 8 63.52919 131541 0.12827072 0.12513399 9 9 64.41341 152492 0.14436482 0.14169811 10 10 66.84620 136200 0.16106678 0.15649254 # ... with 54 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp18_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp18_hilbert_invertion_function ***** Calculate the Target #+name: exp18_target #+header: :var dep0=exp18_cumsum #+header: :var dep1=exp18_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp18_tidying #+header: :var dep0=exp18_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp18_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900T5J.png]] **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp18_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** 9.1 Check which rank has the larger correction #+header: :var dep0=exp18_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.9131876 : 2 62 1.1703797 : 3 63 1.0715776 : 4 64 1.0107468 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. : 32.55 1st Qu.:16.75 1st Qu.: 59.81 Median :32.50 Median : 61.14 Mean :32.50 Mean : 61.80 3rd Qu.:48.25 3rd Qu.: 63.00 Max. :64.00 Max. :103.14 [1] 12.66165 # A tibble: 64 x 2 Rank time 1 31 103.14403 2 59 97.99925 3 46 96.40160 4 44 91.13452 5 3 81.42977 6 20 77.99168 7 1 67.54315 8 36 66.69546 9 56 65.95448 10 39 65.71467 # ... with 54 more rows #+end_example ** FINALLY :EXP20: *** Use the =sfc= branch of alya #+begin_src shell :results output cd ~/misc/ svn checkout svn+ssh://bsc21835@dt01.bsc.es/gpfs/projects/bsc21/svnroot/Alya/branches/sfc alya-bsc-sfc #+end_src Copy-paste: - my modified version of Alya.f90 on top of the alya-bsc-sfc branch. - my modified version of inivar.f90 on top of the alya-bsc-sfc branch The filename should be named: rank-elements.dat - First column: rank - Second column: the relative weight of each partition Separated by one space - All the data about the partition (subdomain) is registered in a file called =def_domain.f90=, within kernel/defmod #+begin_src shell :results output cd ~/misc/ ORIGIN=alya-bsc TARGET=alya-bsc-sfc cp $ORIGIN/Sources/kernel/master/Alya.f90 $TARGET/Sources/kernel/master/ cp $ORIGIN/Sources/services/parall/par_prepro.f90 $TARGET/Sources/services/parall/ cp $ORIGIN/Executables/unix/config.in $TARGET/Executables/unix/ #+end_src #+RESULTS: *** Alya Instrumentation #+begin_src shell :results output cd ~/misc/alya-bsc-sfc/ svn diff #+end_src #+RESULTS: #+begin_example Index: Sources/kernel/master/Alya.f90 =================================================================== --- Sources/kernel/master/Alya.f90 (revision 8442) +++ Sources/kernel/master/Alya.f90 (working copy) @@ -1,3 +1,4 @@ +#include "scorep/SCOREP_User.inc" !> @file Alya.f90 !! @author Guillaume Houzeaux !! @brief Ayla main @@ -20,7 +21,13 @@ use def_master, only : kfl_goblk use def_master, only : kfl_gocou use def_coupli, only : kfl_gozon + use mod_parall, only : PAR_MY_WORLD_RANK implicit none + INTEGER :: iter,ierror + character*100 striter + character*100 iterfile + real :: tnow + real, dimension(2) :: tarray ! ! DLB should be disabled as we only wabnt to activate it for particular loops ! Master does not disble to lend its resources automatically @@ -39,6 +46,10 @@ call Parall(22270_ip) + write(iterfile,'(a,i4.4,a)') 'iterations-', PAR_MY_WORLD_RANK, '.csv' + open(unit=2, file=iterfile) + + iter = 1 optimization: do while ( kfl_goopt == 1 ) call Iniunk() @@ -49,6 +60,11 @@ time: do while ( kfl_gotim == 1 ) call Timste() + call ETIME(tarray, tnow) + write(2,*) tnow, tarray(1), tarray(2), PAR_MY_WORLD_RANK, iter + + write(striter, '(a,i3.3)') 'iter',iter + SCOREP_USER_REGION_BY_NAME_BEGIN(striter, SCOREP_USER_REGION_TYPE_PHASE) reset: do call Begste() @@ -77,6 +93,13 @@ call Endste() + SCOREP_USER_REGION_BY_NAME_END(striter) + call ETIME(tarray, tnow) + write(2,*) tnow, tarray(1), tarray(2), PAR_MY_WORLD_RANK, iter + iter = iter + 1 + + + call Filter(ITASK_ENDTIM) call Output(ITASK_ENDTIM) @@ -91,6 +114,9 @@ end do optimization + close(2) + call Turnof() + end program Alya Index: Sources/services/parall/par_prepro.f90 =================================================================== --- Sources/services/parall/par_prepro.f90 (revision 8442) +++ Sources/services/parall/par_prepro.f90 (working copy) @@ -19,6 +19,7 @@ use mod_memory use mod_par_partit_sfc, only : par_partit_sfc use mod_parall, only : PAR_WORLD_SIZE + use mod_parall, only : PAR_MY_WORLD_RANK use mod_parall, only : PAR_METIS4 use mod_parall, only : PAR_SFC use mod_parall, only : PAR_ORIENTED_BIN @@ -32,6 +33,9 @@ integer(ip) :: npoin_tmp, nelem_tmp, nboun_tmp real(rp) :: time1,time2,time3,time4,time5 character(100) :: messa_integ + character*100 dfile + integer(ip) ii + ! ! Output ! @@ -354,4 +358,20 @@ end if + ! LUCAS + write(dfile,'(a,i4.4,a)') 'domain-', PAR_MY_WORLD_RANK, '.csv' + open(unit=12345, file=dfile) + + write(12345,*) "T1", PAR_MY_WORLD_RANK, nelem, npoin, nboun, npoi3-npoi2 + do ii = 1, nelem + write(12345,*) "T2", PAR_MY_WORLD_RANK, ii, ltype(ii) + end do + do ii = 1, nboun + write(12345,*) "T3", PAR_MY_WORLD_RANK, ii, ltypb(ii) + end do + + close(12345) + ! END LUCAS + + end subroutine par_prepro #+end_example *** Compile the sfc branch (MPI 3.0 + SCOREP 3) #+begin_src shell :results output source ~/spack-ALYA/share/spack/setup-env.sh export PATH=$(spack location -i openmpi)/bin:$PATH #+end_src *** Run the sfc branch with my modifications #+begin_src shell :results output source ~/spack-ALYA/share/spack/setup-env.sh export PATH=$(spack location -i openmpi)/bin:$PATH $(which mpirun) --bind-to core:overload-allowed --report-bindings \ -x SCOREP_TOTAL_MEMORY=4GB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np $(cat ~/machine-file | wc -l) \ -machinefile ~/machine-file \ ~/alya-bsc-sfc/Executables/unix/Alya.x fensap #+end_src *** Extracting information from the partitions #+BEGIN_EXAMPLE character*100 dfile integer(ip) ii ! LUCAS write(dfile,'(a,i4.4,a)') 'domain-', PAR_MY_WORLD_RANK, '.csv' open(unit=12345, file=dfile) write(12345,*) "T1", PAR_MY_WORLD_RANK, nelem, npoin, nboun, npoi3-npoi2 do ii = 1, nelem write(12345,*) "T2", PAR_MY_WORLD_RANK, ii, ltype(ii) end do do ii = 1, nboun write(12345,*) "T3", PAR_MY_WORLD_RANK, ii, ltypb(ii) end do close(12345) ! END LUCAS #+END_EXAMPLE *** Data Transformation **** 0. Experiment directory #+name: exp20_dir #+begin_src shell :results output echo -n "exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480/" #+end_src #+RESULTS: exp20_dir : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480/ **** 1. Read Iteration Timings #+name: exp20_iteration_timings #+header: :var DIR=exp20_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src **** 2. Transform profile.cubex #+name: exp20_cubex_to_open #+header: :var DIR=exp20_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp20_cubex_to_open **** 3. Parse the call tree #+name: exp20_cube_calltree #+header: :var dep0=exp20_cubex_to_open #+header: :var DIR=exp20_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp20_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 4. Enrich the call tree #+name: exp20_enrich #+header: :var CSV=exp20_cube_calltree #+header: :var DIR=exp20_dir #+begin_src R :results output :session :exports both PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`); #+end_src #+RESULTS: exp20_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** 5. The number of elements #+name: exp20_number_of_elements_shell #+header: :var DIR=exp20_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+name: exp20_number_of_elements #+header: :var TABLE=exp20_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp20_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp20_integrate #+header: :var dep1=exp20_enrich #+header: :var dep0=exp20_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Rank != 0) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% left_join(exp.ELEMENTS) %>% select(Phase, Rank, time, NELEM) -> df.INTEGRATED; #+end_src #+RESULTS: : Joining, by = "Rank" **** 7. Do the cumsum #+name: exp20_cumsum #+header: :var dep0=exp20_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Phase, Rank) %>% group_by(Phase) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) %>% ungroup() -> df.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp20_cumsum #+begin_example # A tibble: 128 x 6 Phase Rank time NELEM TIMESUM NELEMSUM 1 1 1 48.16957 143862 0.01204776 0.01562670 2 1 2 68.57628 143921 0.02919947 0.03125982 3 1 3 62.76540 143822 0.04489782 0.04688218 4 1 4 70.52202 143695 0.06253618 0.06249074 5 1 5 51.59148 143675 0.07543980 0.07809713 6 1 6 57.46073 143752 0.08981139 0.09371189 7 1 7 66.35009 143768 0.10640630 0.10932838 8 1 8 68.51840 143792 0.12354354 0.12494748 9 1 9 64.11168 143945 0.13957860 0.14058320 10 1 10 57.19916 144243 0.15388477 0.15625129 # ... with 118 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve #+name: exp20_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp20_hilbert_invertion_function #+name: exp20_target #+header: :var dep0=exp20_cumsum #+header: :var dep1=exp20_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% filter(Phase == 2) %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+RESULTS: exp20_target #+begin_example # A tibble: 64 x 8 Phase Rank time NELEM y x Target.x Target.y 1 2 1 51.55150 143862 0.01265736 0.01562670 0.01834748 0.015625 2 2 2 69.44857 143921 0.02970895 0.03125982 0.03265310 0.031250 3 2 3 70.37532 143822 0.04698809 0.04688218 0.04677993 0.046875 4 2 4 73.12350 143695 0.06494199 0.06249074 0.06036775 0.062500 5 2 5 62.68656 143675 0.08033332 0.07809713 0.07585795 0.078125 6 2 6 68.69897 143752 0.09720087 0.09371189 0.09051732 0.093750 7 2 7 69.42635 143768 0.11424701 0.10932838 0.10486498 0.109375 8 2 8 63.86800 143792 0.12992842 0.12494748 0.12003864 0.125000 9 2 9 53.87067 143945 0.14315520 0.14058320 0.13759219 0.140625 10 2 10 82.17739 144243 0.16333208 0.15625129 0.15075179 0.156250 # ... with 54 more rows #+end_example #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0.5 m2=m1+0.3 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=4, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) + geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure169004sh.png]] #+begin_src R :results output :session :exports both df_cum_inverse(df.CUMSUM.2, yval = 0.02) #+end_src #+RESULTS: : [1] 300 : [1] 0.02 : [1] 5 : x y : 5 0.01334887 0.01937010 : 6 0.01669545 0.02087636 : [1] 0.01474838 **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp20_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: *** Visualization / Analysis **** Rough initial LB Analysis #+header: :var dep1=exp20_points #+header: :var dep0=exp20_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% filter(Rank != 0) %>% filter(Code == "Computation") %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.1) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900R5y.png]] ** 4-node grisou (64 cores + 1 core) with 10 iterations :EXP21: *** Data Transformation **** 0. Experiment directory #+name: exp21_dir #+begin_src shell :results output echo -n "exp_21_grisou_4_manual/scorep-20180125_1431_56251511746269/" #+end_src #+RESULTS: exp21_dir : exp_21_grisou_4_manual/scorep-20180125_1431_56251511746269/ **** 1. Read Iteration Timings #+name: exp21_iteration_timings #+header: :var DIR=exp21_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src **** 2. Transform profile.cubex #+name: exp21_cubex_to_open #+header: :var DIR=exp21_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp21_cubex_to_open **** 3. Parse the call tree #+name: exp21_cube_calltree #+header: :var dep0=exp21_cubex_to_open #+header: :var DIR=exp21_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp21_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 4. Enrich the call tree #+name: exp21_enrich #+header: :var CSV=exp21_cube_calltree #+header: :var DIR=exp21_dir #+begin_src R :results output :session :exports both PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src #+RESULTS: exp21_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+header: :var dep0=exp21_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900F4Q.png]] ***** Calculate the average #+name: exp21_lb_average #+header: :var dep0=exp21_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp21_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** 5. The number of elements on T1 #+name: exp21_number_of_elements_shell #+header: :var DIR=exp21_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+name: exp21_number_of_elements #+header: :var TABLE=exp21_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp21_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp21_integrate #+header: :var dep1=exp21_lb_average #+header: :var dep0=exp21_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp21_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 42.92296 143862 2 2 64.98924 143921 3 3 66.13120 143822 4 4 62.14705 143695 5 5 64.66830 143675 6 6 59.88956 143752 7 7 61.76306 143768 8 8 67.09068 143792 9 9 57.22863 143945 10 10 64.70995 144243 # ... with 54 more rows #+end_example **** 7. Do the cumsum #+name: exp21_cumsum #+header: :var dep0=exp21_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp21.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp21_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 42.92296 143862 0.01094528 0.01562670 2 2 64.98924 143921 0.02751743 0.03125982 3 3 66.13120 143822 0.04438077 0.04688218 4 4 62.14705 143695 0.06022817 0.06249074 5 5 64.66830 143675 0.07671848 0.07809713 6 6 59.88956 143752 0.09199021 0.09371189 7 7 61.76306 143768 0.10773969 0.10932838 8 8 67.09068 143792 0.12484770 0.12494748 9 9 57.22863 143945 0.13944090 0.14058320 10 10 64.70995 144243 0.15594183 0.15625129 # ... with 54 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp21_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp21_hilbert_invertion_function ***** Calculate the Target #+name: exp21_target #+header: :var dep0=exp21_cumsum #+header: :var dep1=exp21_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp21_tidying #+header: :var dep0=exp21_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp21_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900iIJ.png]] **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp21_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** 9.1 Check which rank has the larger correction #+header: :var dep0=exp21_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.9816559 : 2 62 1.5405989 : 3 63 1.1311167 : 4 64 1.0905741 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 32.55414 143674 0.0002265834 30.051996 : 2 62 48.51618 143894 0.0003371661 14.089961 : 3 63 56.97007 143735 0.0003963549 5.636066 : 4 64 57.51256 144024 0.0003993262 5.093579 *** Visualization / Analysis **** Rough initial LB Analysis #+header: :var dep0=exp21_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% filter(Rank != 0) %>% filter(Code == "Computation") %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point(alpha=.2, size=3) + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900YNW.png]] *** Trying to craft a model based on nelem and ltype(elements) **** 0. The number of elements on T2 #+name: exp21_number_of_elements_shell_t2 #+header: :var DIR=exp21_dir #+begin_src shell :results output cd $DIR TEMPFILE=/tmp/xis #$(mktemp) echo "Type,Rank,Element.ID,Element.Type" > $TEMPFILE cat domain-*.csv | grep ^.*T2 | sed -e "s/ */,/g" -e "s/^,//" >> $TEMPFILE echo -n $TEMPFILE #+end_src #+RESULTS: exp21_number_of_elements_shell_t2 : /tmp/xis #+header: :var T2FILE=exp21_number_of_elements_shell_t2 #+begin_src R :results output :session :exports both T2FILE %>% read_csv %>% select(-Type) %>% group_by(Rank, Element.Type) %>% summarize(N=n()) %>% mutate(Element.Type = case_when(Element.Type == 30 ~ "Tetra", Element.Type == 37 ~ "Hexa", Element.Type == 32 ~ "Pyra", Element.Type == 34 ~ "Penta", TRUE ~ "Unknown")) -> exp.T2DATA; exp.T2DATA; #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Type = col_character(), Rank = col_integer(), Element.ID = col_integer(), Element.Type = col_integer() ) # A tibble: 138 x 3 # Groups: Rank [64] Rank Element.Type N 1 1 Tetra 143767 2 1 Pyra 32 3 1 Penta 63 4 2 Tetra 90229 5 2 Pyra 76 6 2 Penta 53616 7 3 Tetra 95492 8 3 Penta 48330 9 4 Tetra 102585 10 4 Penta 41110 # ... with 128 more rows #+end_example #+name: exp21_number_of_elements #+header: :var TABLE=exp21_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src **** 1. Dataset from previous workflow ***** Get the data #+header: :var dep0=exp21_enrich #+header: :var dep1=exp21_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.T2DATA) %>% select(Phase, Rank, time, Element.Type, N) -> exp.DATASET; #+end_src #+RESULTS: : Joining, by = "Rank" #+begin_src R :results output :session :exports both exp.ENRICH %>% left #exp.DATASET %>% # select( # spread(Type, Amount, - #+end_src #+RESULTS: : Error in overscope_eval_next(overscope, expr) : : object 'Element.Type' not found ***** Try to fit #+begin_src R :results output :session :exports both summary(lm(data=exp.DATASET, time ~ N+Element.Type)) #NELEM+NPOIN+NBOUN+NPOI32)) #+end_src #+RESULTS: #+begin_example Call: lm(formula = time ~ Element.Type, data = exp.DATASET) Residuals: Min 1Q Median 3Q Max -39.874 -5.407 3.344 9.236 27.574 Coefficients: Estimate Std. Error t value Pr(>|t|) (Intercept) 6.127e+01 5.416e-01 113.141 < 2e-16 *** Element.TypePyra -9.350e+00 1.473e+00 -6.346 2.99e-10 *** Element.TypeTetra 1.003e-14 7.659e-01 0.000 1 --- Signif. codes: 0 ‘***’ 0.001 ‘**’ 0.01 ‘*’ 0.05 ‘.’ 0.1 ‘ ’ 1 Residual standard error: 13.7 on 1377 degrees of freedom Multiple R-squared: 0.03041, Adjusted R-squared: 0.02901 F-statistic: 21.6 on 2 and 1377 DF, p-value: 5.82e-10 #+end_example ***** Plot the chaos #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session ggplot(data=exp.DATASET, aes(x=NELEM,y=time, color=Rank)) + geom_point() #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900sNQ.png]] ** 4-node grisou (64 cores + 1 core) with 10 iters and rank elements :EXP22: *** Data Transformation **** 0. Experiment directory #+name: exp22_dir #+begin_src shell :results output echo -n "exp_22_grisou_4_manual/scorep-20180125_1509_61786778640531" #+end_src #+RESULTS: exp22_dir : exp_22_grisou_4_manual/scorep-20180125_1509_61786778640531 **** 1. Read Iteration Timings #+name: exp22_iteration_timings #+header: :var DIR=exp22_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src **** 2. Transform profile.cubex #+name: exp22_cubex_to_open #+header: :var DIR=exp22_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp22_cubex_to_open **** 3. Parse the call tree #+name: exp22_cube_calltree #+header: :var dep0=exp22_cubex_to_open #+header: :var DIR=exp22_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp22_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 4. Enrich the call tree #+name: exp22_enrich #+header: :var CSV=exp22_cube_calltree #+header: :var DIR=exp22_dir #+begin_src R :results output :session :exports both PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src #+RESULTS: exp22_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+header: :var dep0=exp22_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900skL.png]] ***** Calculate the average #+name: exp22_lb_average #+header: :var dep0=exp22_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp22_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** 5. The number of elements on T1 #+name: exp22_number_of_elements_shell #+header: :var DIR=exp22_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+name: exp22_number_of_elements #+header: :var TABLE=exp22_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp22_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp22_integrate #+header: :var dep1=exp22_lb_average #+header: :var dep0=exp22_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp22_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 42.92296 143862 2 2 64.98924 143921 3 3 66.13120 143822 4 4 62.14705 143695 5 5 64.66830 143675 6 6 59.88956 143752 7 7 61.76306 143768 8 8 67.09068 143792 9 9 57.22863 143945 10 10 64.70995 144243 # ... with 54 more rows #+end_example **** 7. Do the cumsum #+name: exp22_cumsum #+header: :var dep0=exp22_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp22.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp22_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 60.43743 184387 0.01510068 0.02002865 2 2 68.98525 135390 0.03233709 0.03473510 3 3 62.07687 134635 0.04784740 0.04935954 4 4 61.41787 140751 0.06319305 0.06464832 5 5 66.77389 137213 0.07987694 0.07955279 6 6 60.27102 146712 0.09493604 0.09548907 7 7 69.88611 141375 0.11239754 0.11084562 8 8 63.52919 131541 0.12827072 0.12513399 9 9 64.41341 152492 0.14436482 0.14169811 10 10 66.84620 136200 0.16106678 0.15649254 # ... with 54 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp22_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp22_hilbert_invertion_function ***** Calculate the Target #+name: exp22_target #+header: :var dep0=exp22_cumsum #+header: :var dep1=exp22_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp22_tidying #+header: :var dep0=exp22_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp22_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900k8u.png]] **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp22_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** 9.1 Check which rank has the larger correction #+header: :var dep0=exp22_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank == 30) #+end_src #+RESULTS: : # A tibble: 1 x 2 : Rank Target : : 1 30 1.325439 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd #+end_src #+RESULTS: : [1] 1 : Rank time : Min. : 1.00 Min. :29.42 : 1st Qu.:16.75 1st Qu.:60.40 : Median :32.50 Median :62.84 : Mean :32.50 Mean :62.54 : 3rd Qu.:48.25 3rd Qu.:66.38 : Max. :64.00 Max. :89.45 : [1] 11.35958 *** Visualization / Analysis **** Rough initial LB Analysis #+header: :var dep0=exp22_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% filter(Rank != 0) %>% filter(Code == "Computation") %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point(alpha=.2, size=3) + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900xgb.png]] **** Merge the cumsum of exp21 and exp22 #+header: :var dep0=exp21_cumsum #+header: :var dep1=exp22_cumsum #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1200 :height 800 :session z0 = 0.5 z1 = z0+0.5 df.exp21.CUMSUM %>% mutate(Type = "exp21") %>% bind_rows(df.exp22.CUMSUM %>% mutate(Type = "exp22")) %>% ggplot(aes(x=NELEMSUM, y=TIMESUM)) + geom_line() + geom_point(aes(color=Type, size=time)) + coord_cartesian(xlim=c(z0, z1), ylim=c(z0, z1)) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900M6C.png]] ** 4-node grisou (64 cores + 1 core) 10its, new round :EXP23: *** Data Transformation **** 0. Experiment directory #+name: exp23_dir #+begin_src shell :results output echo -n "exp_23_grisou_4_manual/scorep-20180125_1607_70169635606280" #+end_src #+RESULTS: exp23_dir : exp_23_grisou_4_manual/scorep-20180125_1607_70169635606280 **** 1. Read Iteration Timings #+name: exp23_iteration_timings #+header: :var DIR=exp23_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src **** 2. Transform profile.cubex #+name: exp23_cubex_to_open #+header: :var DIR=exp23_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp23_cubex_to_open **** 3. Parse the call tree #+name: exp23_cube_calltree #+header: :var dep0=exp23_cubex_to_open #+header: :var DIR=exp23_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp23_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 4. Enrich the call tree #+name: exp23_enrich #+header: :var CSV=exp23_cube_calltree #+header: :var DIR=exp23_dir #+begin_src R :results output :session :exports both PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src #+RESULTS: exp23_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+header: :var dep0=exp23_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900TRG.png]] ***** Calculate the average #+name: exp23_lb_average #+header: :var dep0=exp23_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp23_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** 5. The number of elements on T1 #+name: exp23_number_of_elements_shell #+header: :var DIR=exp23_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+name: exp23_number_of_elements #+header: :var TABLE=exp23_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp23_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp23_integrate #+header: :var dep1=exp23_lb_average #+header: :var dep0=exp23_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp23_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 42.92296 143862 2 2 64.98924 143921 3 3 66.13120 143822 4 4 62.14705 143695 5 5 64.66830 143675 6 6 59.88956 143752 7 7 61.76306 143768 8 8 67.09068 143792 9 9 57.22863 143945 10 10 64.70995 144243 # ... with 54 more rows #+end_example **** 7. Do the cumsum #+name: exp23_cumsum #+header: :var dep0=exp23_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp23.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp23_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 60.43743 184387 0.01510068 0.02002865 2 2 68.98525 135390 0.03233709 0.03473510 3 3 62.07687 134635 0.04784740 0.04935954 4 4 61.41787 140751 0.06319305 0.06464832 5 5 66.77389 137213 0.07987694 0.07955279 6 6 60.27102 146712 0.09493604 0.09548907 7 7 69.88611 141375 0.11239754 0.11084562 8 8 63.52919 131541 0.12827072 0.12513399 9 9 64.41341 152492 0.14436482 0.14169811 10 10 66.84620 136200 0.16106678 0.15649254 # ... with 54 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp23_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp23_hilbert_invertion_function ***** Calculate the Target #+name: exp23_target #+header: :var dep0=exp23_cumsum #+header: :var dep1=exp23_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp23_tidying #+header: :var dep0=exp23_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp23_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900k8u.png]] **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp23_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** 9.1 Check which rank has the larger correction #+header: :var dep0=exp23_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.9190534 : 2 62 1.0775470 : 3 63 1.1810391 : 4 64 1.0176706 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. :31.29 1st Qu.:16.75 1st Qu.:61.21 Median :32.50 Median :62.77 Mean :32.50 Mean :62.20 3rd Qu.:48.25 3rd Qu.:64.36 Max. :64.00 Max. :94.47 [1] 9.980286 # A tibble: 64 x 2 Rank time 1 30 94.46917 2 63 91.98761 3 41 79.08293 4 21 76.44483 5 62 71.74396 6 29 70.84170 7 43 70.29370 8 42 69.40785 9 17 67.79000 10 4 66.53386 # ... with 54 more rows #+end_example *** Visualization / Analysis **** Merge the cumsum of exp21 and exp22 and exp23 #+header: :var dep0=exp21_cumsum #+header: :var dep1=exp22_cumsum #+header: :var dep1=exp23_cumsum #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1200 :height 800 :session oz0 = 0 z1 = z0+1 df.exp21.CUMSUM %>% mutate(Type = "exp21") %>% bind_rows(df.exp22.CUMSUM %>% mutate(Type = "exp22")) %>% bind_rows(df.exp23.CUMSUM %>% mutate(Type = "exp23")) %>% group_by(Type) %>% mutate(Rank.Worst = (time == max(time))) %>% ggplot(aes(x=NELEMSUM, y=TIMESUM)) + geom_line() + geom_point(aes(color=Type, size=time, shape=Rank.Worst)) + coord_cartesian(xlim=c(z0, z1), ylim=c(z0, z1)) + theme_bw(base_size = 22) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900PxZ.png]] ** 4-node grisou (64 cores + 1 core) 5its (STABLE: HT, TB) :EXP24: *** Differences - New round - No hyperthreading - No turboboost - MPI bind to core *** Data Transformation **** 0. Experiment directory #+name: exp24_dir #+begin_src shell :results output echo -n "exp_24_grisou_4_manual/scorep-20180125_1706_78570606376628/"; #+end_src #+RESULTS: exp24_dir : exp_24_grisou_4_manual/scorep-20180125_1706_78570606376628/ **** 1. Read Iteration Timings #+name: exp24_iteration_timings #+header: :var DIR=exp24_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src **** 2. Transform profile.cubex #+name: exp24_cubex_to_open #+header: :var DIR=exp24_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp24_cubex_to_open **** 3. Parse the call tree #+name: exp24_cube_calltree #+header: :var dep0=exp24_cubex_to_open #+header: :var DIR=exp24_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp24_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 4. Enrich the call tree #+name: exp24_enrich #+header: :var CSV=exp24_cube_calltree #+header: :var DIR=exp24_dir #+begin_src R :results output :session :exports both PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src #+RESULTS: exp24_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+header: :var dep0=exp24_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-27902p-4/figure27902TuK.png]] ***** Calculate the average #+name: exp24_lb_average #+header: :var dep0=exp24_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp24_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** 5. The number of elements on T1 #+name: exp24_number_of_elements_shell #+header: :var DIR=exp24_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+name: exp24_number_of_elements #+header: :var TABLE=exp24_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp24_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp24_integrate #+header: :var dep1=exp24_lb_average #+header: :var dep0=exp24_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp24_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 42.92296 143862 2 2 64.98924 143921 3 3 66.13120 143822 4 4 62.14705 143695 5 5 64.66830 143675 6 6 59.88956 143752 7 7 61.76306 143768 8 8 67.09068 143792 9 9 57.22863 143945 10 10 64.70995 144243 # ... with 54 more rows #+end_example **** 7. Do the cumsum #+name: exp24_cumsum #+header: :var dep0=exp24_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp24.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp24_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 60.43743 184387 0.01510068 0.02002865 2 2 68.98525 135390 0.03233709 0.03473510 3 3 62.07687 134635 0.04784740 0.04935954 4 4 61.41787 140751 0.06319305 0.06464832 5 5 66.77389 137213 0.07987694 0.07955279 6 6 60.27102 146712 0.09493604 0.09548907 7 7 69.88611 141375 0.11239754 0.11084562 8 8 63.52919 131541 0.12827072 0.12513399 9 9 64.41341 152492 0.14436482 0.14169811 10 10 66.84620 136200 0.16106678 0.15649254 # ... with 54 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp24_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp24_hilbert_invertion_function ***** Calculate the Target #+name: exp24_target #+header: :var dep0=exp24_cumsum #+header: :var dep1=exp24_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp24_tidying #+header: :var dep0=exp24_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp24_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900k8u.png]] **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp24_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** 9.1 Check which rank has the larger correction #+header: :var dep0=exp24_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 1.0533256 : 2 62 0.7642869 : 3 63 0.7402923 : 4 64 0.8513753 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. : 30.46 1st Qu.:16.75 1st Qu.: 59.09 Median :32.50 Median : 63.61 Mean :32.50 Mean : 63.07 3rd Qu.:48.25 3rd Qu.: 71.25 Max. :64.00 Max. :104.52 [1] 15.64072 # A tibble: 64 x 2 Rank time 1 63 104.52326 2 39 89.81473 3 1 88.79020 4 19 85.54385 5 25 84.91805 6 35 84.68214 7 18 81.35206 8 15 81.18039 9 34 76.98144 10 26 76.94032 # ... with 54 more rows #+end_example ** 4-node grisou (64 cores + 1 core) 1its, new round :EXP25: *** Differences The same as exp24 *** Data Transformation **** 0. Experiment directory #+name: exp25_dir #+begin_src shell :results output echo -n "exp_25_grisou_4_manual/scorep-20180125_1730_82025208841684/" #+end_src #+RESULTS: exp25_dir : exp_25_grisou_4_manual/scorep-20180125_1730_82025208841684/ **** 1. Read Iteration Timings #+name: exp25_iteration_timings #+header: :var DIR=exp25_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src **** 2. Transform profile.cubex #+name: exp25_cubex_to_open #+header: :var DIR=exp25_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp25_cubex_to_open **** 3. Parse the call tree #+name: exp25_cube_calltree #+header: :var dep0=exp25_cubex_to_open #+header: :var DIR=exp25_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp25_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 4. Enrich the call tree #+name: exp25_enrich #+header: :var CSV=exp25_cube_calltree #+header: :var DIR=exp25_dir #+begin_src R :results output :session :exports both PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src #+RESULTS: exp25_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+header: :var dep0=exp25_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900TFI.png]] ***** Calculate the average #+name: exp25_lb_average #+header: :var dep0=exp25_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp25_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** 5. The number of elements on T1 #+name: exp25_number_of_elements_shell #+header: :var DIR=exp25_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+name: exp25_number_of_elements #+header: :var TABLE=exp25_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp25_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp25_integrate #+header: :var dep1=exp25_lb_average #+header: :var dep0=exp25_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp25_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 42.92296 143862 2 2 64.98924 143921 3 3 66.13120 143822 4 4 62.14705 143695 5 5 64.66830 143675 6 6 59.88956 143752 7 7 61.76306 143768 8 8 67.09068 143792 9 9 57.22863 143945 10 10 64.70995 144243 # ... with 54 more rows #+end_example **** 7. Do the cumsum #+name: exp25_cumsum #+header: :var dep0=exp25_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp25.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp25_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 60.43743 184387 0.01510068 0.02002865 2 2 68.98525 135390 0.03233709 0.03473510 3 3 62.07687 134635 0.04784740 0.04935954 4 4 61.41787 140751 0.06319305 0.06464832 5 5 66.77389 137213 0.07987694 0.07955279 6 6 60.27102 146712 0.09493604 0.09548907 7 7 69.88611 141375 0.11239754 0.11084562 8 8 63.52919 131541 0.12827072 0.12513399 9 9 64.41341 152492 0.14436482 0.14169811 10 10 66.84620 136200 0.16106678 0.15649254 # ... with 54 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp25_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp25_hilbert_invertion_function ***** Calculate the Target #+name: exp25_target #+header: :var dep0=exp25_cumsum #+header: :var dep1=exp25_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp25_tidying #+header: :var dep0=exp25_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp25_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900k8u.png]] **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp25_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** 9.1 Check which rank has the larger correction #+header: :var dep0=exp25_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.8920903 : 2 62 1.1812863 : 3 63 1.0454509 : 4 64 0.9953072 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 55.86152 151485 0.0003687594 6.125037 : 2 62 38.89725 109918 0.0003538752 23.089306 : 3 63 48.54840 106440 0.0004561104 13.438163 : 4 64 52.56137 122507 0.0004290479 9.425188 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. : 35.71 1st Qu.:16.75 1st Qu.: 51.84 Median :32.50 Median : 59.71 Mean :32.50 Mean : 61.99 3rd Qu.:48.25 3rd Qu.: 70.18 Max. :64.00 Max. :101.61 [1] 14.53319 # A tibble: 64 x 2 Rank time 1 33 101.60772 2 57 96.35507 3 20 95.07653 4 46 93.58917 5 50 89.33524 6 56 85.84247 7 38 85.29922 8 51 84.17110 9 12 74.24943 10 48 74.22989 # ... with 54 more rows #+end_example *** Visualization / Analysis **** Merge the cumsum of exp24 and exp25 #+header: :var dep0=exp24_cumsum #+header: :var dep1=exp25_cumsum #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1200 :height 800 :session oz0 = 0 z1 = z0+1 df.exp24.CUMSUM %>% mutate(Type = "exp24") %>% bind_rows(df.exp25.CUMSUM %>% mutate(Type = "exp25")) %>% group_by(Type) %>% mutate(Rank.Worst = (time == max(time)), Ynonnormalized = cumsum(time)) %>% ggplot(aes(x=NELEMSUM, y=Ynonnormalized)) + geom_line() + geom_point(aes(color=Type, size=time, shape=Rank.Worst)) + coord_cartesian(xlim=c(z0, z1)) + #, ylim=c(z0, z1)) + theme_bw(base_size = 22) #+end_src #+RESULTS: [[file:/tmp/babel-27902p-4/figure279026aF.png]] ** 4-node grisou (64 cores + 1 core) 5its, new round :EXP26: *** Data Transformation **** 0. Experiment directory #+name: exp26_dir #+begin_src shell :results output echo -n "exp_26_grisou_4_manual/scorep-20180125_1752_85193983406456" #+end_src #+RESULTS: exp26_dir : exp_26_grisou_4_manual/scorep-20180125_1752_85193983406456 **** 1. Read Iteration Timings #+name: exp26_iteration_timings #+header: :var DIR=exp26_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src **** 2. Transform profile.cubex #+name: exp26_cubex_to_open #+header: :var DIR=exp26_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp26_cubex_to_open **** 3. Parse the call tree #+name: exp26_cube_calltree #+header: :var dep0=exp26_cubex_to_open #+header: :var DIR=exp26_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp26_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 4. Enrich the call tree #+name: exp26_enrich #+header: :var CSV=exp26_cube_calltree #+header: :var DIR=exp26_dir #+begin_src R :results output :session :exports both PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src #+RESULTS: exp26_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+header: :var dep0=exp26_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900Vtv.png]] ***** Calculate the average #+name: exp26_lb_average #+header: :var dep0=exp26_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp26_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** 5. The number of elements on T1 #+name: exp26_number_of_elements_shell #+header: :var DIR=exp26_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+name: exp26_number_of_elements #+header: :var TABLE=exp26_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp26_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp26_integrate #+header: :var dep1=exp26_lb_average #+header: :var dep0=exp26_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp26_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 42.92296 143862 2 2 64.98924 143921 3 3 66.13120 143822 4 4 62.14705 143695 5 5 64.66830 143675 6 6 59.88956 143752 7 7 61.76306 143768 8 8 67.09068 143792 9 9 57.22863 143945 10 10 64.70995 144243 # ... with 54 more rows #+end_example **** 7. Do the cumsum #+name: exp26_cumsum #+header: :var dep0=exp26_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp26.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp26_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 60.43743 184387 0.01510068 0.02002865 2 2 68.98525 135390 0.03233709 0.03473510 3 3 62.07687 134635 0.04784740 0.04935954 4 4 61.41787 140751 0.06319305 0.06464832 5 5 66.77389 137213 0.07987694 0.07955279 6 6 60.27102 146712 0.09493604 0.09548907 7 7 69.88611 141375 0.11239754 0.11084562 8 8 63.52919 131541 0.12827072 0.12513399 9 9 64.41341 152492 0.14436482 0.14169811 10 10 66.84620 136200 0.16106678 0.15649254 # ... with 54 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp26_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp26_hilbert_invertion_function ***** Calculate the Target #+name: exp26_target #+header: :var dep0=exp26_cumsum #+header: :var dep1=exp26_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp26_tidying #+header: :var dep0=exp26_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp26_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900T5J.png]] **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp26_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** 9.1 Check which rank has the larger correction #+header: :var dep0=exp26_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.9131876 : 2 62 1.1703797 : 3 63 1.0715776 : 4 64 1.0107468 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. : 32.55 1st Qu.:16.75 1st Qu.: 59.81 Median :32.50 Median : 61.14 Mean :32.50 Mean : 61.80 3rd Qu.:48.25 3rd Qu.: 63.00 Max. :64.00 Max. :103.14 [1] 12.66165 # A tibble: 64 x 2 Rank time 1 31 103.14403 2 59 97.99925 3 46 96.40160 4 44 91.13452 5 3 81.42977 6 20 77.99168 7 1 67.54315 8 36 66.69546 9 56 65.95448 10 39 65.71467 # ... with 54 more rows #+end_example *** Visualization / Analysis **** Merge the cumsum of exp24 and exp25 and exp26 #+header: :var dep0=exp24_cumsum #+header: :var dep1=exp25_cumsum #+header: :var dep2=exp26_cumsum #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1200 :height 800 :session z0 = 0 z1 = z0+1 df.exp24.CUMSUM %>% mutate(Type = "exp24") %>% bind_rows(df.exp25.CUMSUM %>% mutate(Type = "exp25")) %>% bind_rows(df.exp26.CUMSUM %>% mutate(Type = "exp26")) %>% filter(Type != "exp24") %>% group_by(Type) %>% mutate(Rank.Worst = (time == max(time)), Ynonnormalized = cumsum(time)) %>% ggplot(aes(x=NELEMSUM, y=Ynonnormalized)) + geom_line() + geom_point(aes(color=Type, size=time, shape=Rank.Worst)) + coord_cartesian(xlim=c(z0, z1)) + #, ylim=c(z0, z1)) + theme_bw(base_size = 22) #+end_src #+RESULTS: [[file:/tmp/babel-27902p-4/figure279028vF.png]] #+begin_src R :results output graphics :file img/exp25-exp26-small_big_outliers_are_together.png :exports both :width 1200 :height 800 :session library(ggrepel); z0 = 0 z1 = z0+1 df.exp24.CUMSUM %>% mutate(Type = "exp24") %>% bind_rows(df.exp25.CUMSUM %>% mutate(Type = "exp25")) %>% bind_rows(df.exp26.CUMSUM %>% mutate(Type = "exp26")) %>% filter(Type == "exp26") %>% group_by(Type) %>% mutate(Rank.Worst = (time > 0.9*max(time) | time <= 1.2*min(time)), Ynonnormalized = cumsum(time)) -> t t %>% filter(Rank.Worst) -> z; t %>% ggplot(aes(x=NELEMSUM, y=TIMESUM)) + geom_line(aes(group=Type), alpha=.6) + geom_point(aes(color=as.factor(Rank%%8), shape=Type, size=time, alpha=Rank.Worst)) + coord_cartesian(xlim=c(z0, z1), ylim=c(z0, z1)) + theme_bw(base_size = 16) + scale_color_brewer(palette = "Set1") + geom_hline(data=tibble(VAL=1:NP), aes(yintercept=VAL/NP, color=as.factor(VAL%%8))) + geom_text_repel(data=z, force=15, size=4, box.padding=1, max_iter=100, aes(label=paste0( "Rank:", Rank, "\nTime:", round(time,2)))) + xlim(z0, z1) #+end_src #+RESULTS: [[file:img/exp25-exp26-small_big_outliers_are_together.png]] Intepretation - Extremes are together #+begin_src R :results output graphics :file img/exp25-exp26-cost_of_region_is_surprising.png :exports both :width 1200 :height 800 :session library(ggrepel); z0 = 0.8 z1 = z0+0.2 df.exp24.CUMSUM %>% mutate(Type = "exp24") %>% bind_rows(df.exp25.CUMSUM %>% mutate(Type = "exp25")) %>% bind_rows(df.exp26.CUMSUM %>% mutate(Type = "exp26")) %>% filter(Type != "exp24") %>% group_by(Type) %>% mutate(Rank.Worst = (time > 0.9*max(time) | time <= 1.2*min(time)), Ynonnormalized = cumsum(time)) -> t t %>% ggplot(aes(x=NELEMSUM, y=TIMESUM)) + geom_line(aes(group=Type), alpha=.6) + geom_point(aes(color=as.factor(Rank%%8), shape=Type, size=time)) + coord_cartesian(xlim=c(z0, z1), ylim=c(z0, z1)) + theme_bw(base_size = 16) + scale_color_brewer(palette = "Set1") + geom_hline(data=tibble(VAL=1:NP), aes(yintercept=VAL/NP, color=as.factor(VAL%%8))) + geom_text_repel(force=30, aes(label=paste0( "Rank:", Rank, "\nTime:", round(time,2)))) + xlim(z0, z1) #+end_src #+RESULTS: [[file:img/exp25-exp26-cost_of_region_is_surprising.png]] Interpretation: - Rank 59 of exp26 lasts 98 seconds while the Rank 58 of exp25 lasts 42 seconds and both occupy almost the same region of elements (look to the left). - Rank 57 of exp25 lasts 96 seconds while the rank 58 of exp26 lasts 32 seconds and both occupy a similar region (large overlap on X from the previous point to the left following the line). - This is surprising, we should run multiple experiments with the same distribution to confirm that those regions are indeed very costly. #+begin_src R :results output graphics :file img/exp25-exp26-cost_of_region_is_surprising_2.png :exports both :width 1200 :height 800 :session library(ggrepel); z0 = 0.65 z1 = z0+0.2 df.exp24.CUMSUM %>% mutate(Type = "exp24") %>% bind_rows(df.exp25.CUMSUM %>% mutate(Type = "exp25")) %>% bind_rows(df.exp26.CUMSUM %>% mutate(Type = "exp26")) %>% filter(Type != "exp24") %>% group_by(Type) %>% mutate(Rank.Worst = (time > 0.9*max(time) | time <= 1.2*min(time)), Ynonnormalized = cumsum(time)) -> t t %>% ggplot(aes(x=NELEMSUM, y=TIMESUM)) + geom_line(aes(group=Type), alpha=.6) + geom_point(aes(color=as.factor(Rank%%8), shape=Type, size=time)) + coord_cartesian(xlim=c(z0, z1), ylim=c(z0, z1)) + theme_bw(base_size = 16) + scale_color_brewer(palette = "Set1") + geom_hline(data=tibble(VAL=1:NP), aes(yintercept=VAL/NP, color=as.factor(VAL%%8))) + geom_text_repel(force=30, aes(label=paste0( "Rank:", Rank, "\nTime:", round(time,2)))) + xlim(z0, z1) #+end_src #+RESULTS: [[file:img/exp25-exp26-cost_of_region_is_surprising_2.png]] Interpretation: - Rank 48 exp25 (red) takes 74 seconds while the Rank 47 of exp26 (very small pink) occupies almost the same region and takes only 33 seconds. The small additional region (non overlapping between them) is therefore extremely costly. ** 4-node grisou (64 cores + 1 core) 10its, check stability :EXP27: *** Goal for next - Run two executions with exactly the same weight in rank-elements.csv - Weight will be 1 so all ranks get the same load *** Initial rank-elements.dat #+begin_src shell :results output for i in $(seq 1 64); do echo $i 1 done > rank-elements.dat #+end_src #+RESULTS: *** Verify contents of the four executions We should use all but the RANK.HETER whose rank-elements.dat was not 1. #+begin_src shell :results output du -h --max-depth=1 exp_27_grisou_4_manual/exp_27 #+end_src #+RESULTS: : 395M exp_27_grisou_4_manual/exp_27/scorep-20180126_1257_27316133992896 : 395M exp_27_grisou_4_manual/exp_27/RANK.HETER_scorep-20180126_1152_17923889031732 : 395M exp_27_grisou_4_manual/exp_27/scorep-20180126_1506_45830289772336 : 395M exp_27_grisou_4_manual/exp_27/scorep-20180126_1217_21570417239832 : 395M exp_27_grisou_4_manual/exp_27/scorep-20180126_1440_42090372406932 : 2.0G exp_27_grisou_4_manual/exp_27 *** Data Transformation **** 1. Read Iteration Timings #+name: exp27_iteration_timings #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src **** 2. Transform profile.cubex #+name: exp27_current_dir #+begin_src R :results verbatim :session :exports both print(DIR) #+end_src #+RESULTS: exp27_current_dir : exp_27_grisou_4_manual/exp_27/scorep-20180126_1257_27316133992896/ #+name: exp27_cubex_to_open #+header: :var DIR=exp27_current_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp27_cubex_to_open **** 3. Parse the call tree #+name: exp27_cube_calltree #+header: :var dep0=exp27_cubex_to_open #+header: :var DIR=exp27_current_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp27_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 4. Enrich the call tree #+name: exp27_enrich #+header: :var CSV=exp27_cube_calltree #+header: :var DIR=exp27_current_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src #+RESULTS: exp27_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+header: :var dep0=exp27_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-27902p-4/figure27902m1n.png]] ***** Calculate the average #+name: exp27_lb_average #+header: :var dep0=exp27_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp27_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** 5. The number of elements on T1 #+name: exp27_number_of_elements_shell #+header: :var DIR=exp27_current_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+name: exp27_number_of_elements #+header: :var TABLE=exp27_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp27_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example **** 6. Integrate number of elements with compute cost #+name: exp27_integrate #+header: :var dep1=exp27_lb_average #+header: :var dep0=exp27_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp27_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 42.92296 143862 2 2 64.98924 143921 3 3 66.13120 143822 4 4 62.14705 143695 5 5 64.66830 143675 6 6 59.88956 143752 7 7 61.76306 143768 8 8 67.09068 143792 9 9 57.22863 143945 10 10 64.70995 144243 # ... with 54 more rows #+end_example **** 7. Do the cumsum #+name: exp27_cumsum #+header: :var dep0=exp27_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp27.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp27_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 60.43743 184387 0.01510068 0.02002865 2 2 68.98525 135390 0.03233709 0.03473510 3 3 62.07687 134635 0.04784740 0.04935954 4 4 61.41787 140751 0.06319305 0.06464832 5 5 66.77389 137213 0.07987694 0.07955279 6 6 60.27102 146712 0.09493604 0.09548907 7 7 69.88611 141375 0.11239754 0.11084562 8 8 63.52919 131541 0.12827072 0.12513399 9 9 64.41341 152492 0.14436482 0.14169811 10 10 66.84620 136200 0.16106678 0.15649254 # ... with 54 more rows #+end_example **** 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp27_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp27_hilbert_invertion_function ***** Calculate the Target #+name: exp27_target #+header: :var dep0=exp27_cumsum #+header: :var dep1=exp27_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp27_tidying #+header: :var dep0=exp27_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp27_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900T5J.png]] **** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp27_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** 9.1 Check which rank has the larger correction #+header: :var dep0=exp27_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.9131876 : 2 62 1.1703797 : 3 63 1.0715776 : 4 64 1.0107468 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. : 32.55 1st Qu.:16.75 1st Qu.: 59.81 Median :32.50 Median : 61.14 Mean :32.50 Mean : 61.80 3rd Qu.:48.25 3rd Qu.: 63.00 Max. :64.00 Max. :103.14 [1] 12.66165 # A tibble: 64 x 2 Rank time 1 31 103.14403 2 59 97.99925 3 46 96.40160 4 44 91.13452 5 3 81.42977 6 20 77.99168 7 1 67.54315 8 36 66.69546 9 56 65.95448 10 39 65.71467 # ... with 54 more rows #+end_example *** Code to get the four runs together in the same data frame Run this once #+begin_src R :results output :session :exports both exp27 = list() #+end_src #+RESULTS: Get one of the following list at each time #+begin_src R :results output :session :exports both DIR=list.files("exp_27_grisou_4_manual/exp_27", pattern="^scorep", full.names=TRUE)[[4]] DIR #+end_src #+RESULTS: : [1] "exp_27_grisou_4_manual/exp_27/scorep-20180126_1506_45830289772336" For every time you defined DIR above, run the code below #+header: :var dep0=exp27_enrich #+header: :var dep1=exp27_number_of_elements #+begin_src R :results output :session :exports both DIR position = length(exp27) + 1 exp27[position] <- list(exp.ENRICH %>% left_join(exp.ELEMENTS) %>% mutate(Case = position)); length(exp27); #+end_src #+RESULTS: : [1] "exp_27_grisou_4_manual/exp_27/scorep-20180126_1506_45830289772336" : Joining, by = "Rank" : [1] 4 #+begin_src R :results output :session :exports both exp.ALL <- do.call("bind_rows", exp27) %>% mutate(Case = as.factor(Case)) exp.ALL #+end_src #+RESULTS: #+begin_example # A tibble: 2,560 x 21 Phase Code ID Rank visits time min_time max_time bytes_put 1 1 Computation 14 1 1 26.89853 138.7910 138.7910 0 2 1 Computation 14 2 1 41.27433 138.7933 138.7933 0 3 1 Computation 14 3 1 39.85998 138.7941 138.7941 0 4 1 Computation 14 4 1 38.27706 138.7927 138.7927 0 5 1 Computation 14 5 1 38.98377 138.7940 138.7940 0 6 1 Computation 14 6 1 37.20629 138.7926 138.7926 0 7 1 Computation 14 7 1 40.47147 138.7942 138.7942 0 8 1 Computation 14 8 1 40.49991 138.7933 138.7933 0 9 1 Computation 14 9 1 35.34918 138.7931 138.7931 0 10 1 Computation 14 10 1 40.26140 138.7932 138.7932 0 # ... with 2,550 more rows, and 12 more variables: bytes_get , # ALLOCATION_SIZE , DEALLOCATION_SIZE , bytes_leaked , # maximum_heap_memory_allocated , bytes_sent , # bytes_received , NELEM , NPOIN , NBOUN , NPOI32 , # Case #+end_example *** People are anxious, so let's see the result (Rough initial LB Analysis \times 4) #+begin_src R :results output graphics :file img/exp27_stability_check.png :exports both :width 1000 :height 800 :session exp.ALL %>% group_by(Phase, Rank, Code, Case) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) + facet_wrap(~Case) #+end_src #+RESULTS: [[file:img/exp27_stability_check.png]] #+begin_src R :results output graphics :file img/exp27_stability_check_v2_phase1.png :exports both :width 1000 :height 800 :session exp.ALL %>% filter(Phase == 1) %>% group_by(Phase, Rank, Code) %>% summarize(Sum = mean(time), SE=3*sd(time)/sqrt(n())) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=Rank, y=Sum)) + geom_point(alpha=.5) + geom_line(aes(group=Rank), alpha=.2) + geom_errorbar(aes(ymin=Sum-SE, ymax=Sum+SE)) + ylab("Mean") + theme_bw(base_size=18) + facet_wrap(~Phase) #+end_src #+RESULTS: [[file:img/exp27_stability_check_v2_phase1.png]] #+begin_src R :results output graphics :file img/exp27_stability_check_v2.png :exports both :width 1000 :height 800 :session exp.ALL %>% group_by(Phase, Rank, Code) %>% summarize(Sum = mean(time), SE=3*sd(time)/sqrt(n())) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=Rank, y=Sum)) + geom_point(alpha=.5) + geom_line(aes(group=Rank), alpha=.2) + geom_errorbar(aes(ymin=Sum-SE, ymax=Sum+SE), width=.5) + ylab("Mean") + theme_bw(base_size=18) + facet_wrap(~Phase) #+end_src #+RESULTS: [[file:img/exp27_stability_check_v2.png]] *** Textual check just to be sure #+begin_src R :results output :session :exports both exp.ALL %>% filter(Rank == 27) %>% filter(Phase == 2) %>% pull(time) %>% unique #+end_src #+RESULTS: : [1] 35.57875 35.51689 36.33292 36.20780 *** Save the data :ATTACH: :PROPERTIES: :Attachments: exp27_stability_check.csv.gz :ID: 39cb473a-83fa-4019-a2fe-9aea7e4cb373 :END: #+begin_src R :results output :session :exports both write_csv(exp.ALL, "exp27_stability_check.csv") #+end_src #+RESULTS: ** 15-node chetemi (300 cores) :EXP28: *** Goals 1. Finer partitioner mesh #+BEGIN_EXAMPLE Sources/kernel/parall/mod_par_partit_sfc.f90 #+END_EXAMPLE Change =DIM_BIN_CORE= from /128_ip/ to /256_ip/ 2. Check computing stability - Run 5 times with /rank-elements.dat/ set to 1 3. Try to automatize the refinement *** Check Alya modifications of the =sfc= branch #+begin_src shell :results output :dir /ssh:lille.g5k:./alya-bsc-sfc/ svn diff #+end_src #+RESULTS: #+begin_example Index: Sources/kernel/coupli/mod_commdom_alya.f90 =================================================================== --- Sources/kernel/coupli/mod_commdom_alya.f90 (revision 8445) +++ Sources/kernel/coupli/mod_commdom_alya.f90 (working copy) @@ -485,8 +485,8 @@ ! | |_ moduls(ITASK_TIMSTE) ! | |_ setgts(2_ip) <-- minimum of critical time steps. NUNCA PASA POR MODULS!! ! |_Begste <-- aqui ya se tiene el minimo general de todos los modulos!! - CPLNG%sendrecv(1,6) = (current_task==ITASK_BEGSTE).and.( current_when==ITASK_BEFORE).and.& ! \ - ( iblok==1 ).and.(CPLNG%send_modul_i==modul ) ! |__ 2014Dic10 + !CPLNG%sendrecv(1,6) = (current_task==ITASK_BEGSTE).and.( current_when==ITASK_BEFORE).and.& ! \ + ! ( iblok==1 ).and.(CPLNG%send_modul_i==modul ) ! |__ 2014Dic10 CPLNG%sendrecv(2,6) = (current_task==ITASK_BEGSTE).and.( current_when==ITASK_BEFORE).and.& ! | ( iblok==1 ).and.(CPLNG%send_modul_j==modul) ! / Index: Sources/kernel/master/Alya.f90 =================================================================== --- Sources/kernel/master/Alya.f90 (revision 8445) +++ Sources/kernel/master/Alya.f90 (working copy) @@ -1,3 +1,4 @@ +#include "scorep/SCOREP_User.inc" !> @file Alya.f90 !! @author Guillaume Houzeaux !! @brief Ayla main @@ -20,7 +21,13 @@ use def_master, only : kfl_goblk use def_master, only : kfl_gocou use def_coupli, only : kfl_gozon + use mod_parall, only : PAR_MY_WORLD_RANK implicit none + INTEGER :: iter,ierror + character*100 striter + character*100 iterfile + real :: tnow + real, dimension(2) :: tarray ! ! DLB should be disabled as we only wabnt to activate it for particular loops ! Master does not disble to lend its resources automatically @@ -39,6 +46,10 @@ call Parall(22270_ip) + write(iterfile,'(a,i4.4,a)') 'iterations-', PAR_MY_WORLD_RANK, '.csv' + open(unit=2, file=iterfile) + + iter = 1 optimization: do while ( kfl_goopt == 1 ) call Iniunk() @@ -49,6 +60,11 @@ time: do while ( kfl_gotim == 1 ) call Timste() + call ETIME(tarray, tnow) + write(2,*) tnow, tarray(1), tarray(2), PAR_MY_WORLD_RANK, iter + + write(striter, '(a,i3.3)') 'iter',iter + SCOREP_USER_REGION_BY_NAME_BEGIN(striter, SCOREP_USER_REGION_TYPE_PHASE) reset: do call Begste() @@ -77,6 +93,13 @@ call Endste() + SCOREP_USER_REGION_BY_NAME_END(striter) + call ETIME(tarray, tnow) + write(2,*) tnow, tarray(1), tarray(2), PAR_MY_WORLD_RANK, iter + iter = iter + 1 + + + call Filter(ITASK_ENDTIM) call Output(ITASK_ENDTIM) @@ -91,6 +114,9 @@ end do optimization + close(2) + call Turnof() + end program Alya Index: Sources/kernel/parall/mod_par_partit_sfc.f90 =================================================================== --- Sources/kernel/parall/mod_par_partit_sfc.f90 (revision 8445) +++ Sources/kernel/parall/mod_par_partit_sfc.f90 (working copy) @@ -36,7 +36,7 @@ implicit none - integer(ip),PARAMETER :: DIM_BIN_CORE = 128_ip ! Number of boxes per direction per partitioning process + integer(ip),PARAMETER :: DIM_BIN_CORE = 256_ip ! Number of boxes per direction per partitioning process integer(ip),PARAMETER :: VISUALIZATION = 0_ip ! Do you want to visualize: 1=weight, 2=partition integer(ip),PARAMETER :: CRITERIA = 1_ip ! Partition criteria: 0=nodes, 1=elements, 2=weigh.elem.(gauss points) ! 3=weigh. elem. (entries) Index: Sources/services/parall/par_prepro.f90 =================================================================== --- Sources/services/parall/par_prepro.f90 (revision 8445) +++ Sources/services/parall/par_prepro.f90 (working copy) @@ -19,6 +19,7 @@ use mod_memory use mod_par_partit_sfc, only : par_partit_sfc use mod_parall, only : PAR_WORLD_SIZE + use mod_parall, only : PAR_MY_WORLD_RANK use mod_parall, only : PAR_METIS4 use mod_parall, only : PAR_SFC use mod_parall, only : PAR_ORIENTED_BIN @@ -32,6 +33,9 @@ integer(ip) :: npoin_tmp, nelem_tmp, nboun_tmp real(rp) :: time1,time2,time3,time4,time5 character(100) :: messa_integ + character*100 dfile + integer(ip) ii + ! ! Output ! @@ -354,4 +358,20 @@ end if + ! LUCAS + write(dfile,'(a,i4.4,a)') 'domain-', PAR_MY_WORLD_RANK, '.csv' + open(unit=12345, file=dfile) + + write(12345,*) "T1", PAR_MY_WORLD_RANK, nelem, npoin, nboun, npoi3-npoi2 + do ii = 1, nelem + write(12345,*) "T2", PAR_MY_WORLD_RANK, ii, ltype(ii) + end do + do ii = 1, nboun + write(12345,*) "T3", PAR_MY_WORLD_RANK, ii, ltypb(ii) + end do + + close(12345) + ! END LUCAS + + end subroutine par_prepro Index: Utils/user/alya2pos/compile.sh =================================================================== --- Utils/user/alya2pos/compile.sh (revision 8445) +++ Utils/user/alya2pos/compile.sh (working copy) @@ -1,27 +1,27 @@ -ifort -c -traceback -O3 -fpp def_kintyp.f90 -o def_kintyp.o -ifort -c -traceback -O3 def_elmtyp.f90 -o def_elmtyp.o -ifort -c -traceback -O3 def_inpout.f90 -o def_inpout.o -ifort -c -traceback -O3 gidele.f90 -o gidele.o -ifort -c -traceback -O3 gidres_he.f90 -o gidres_he.o -ifort -c -traceback -O3 gidres_gp.f90 -o gidres_gp.o -ifort -c -traceback -O3 elmtyp.f90 -o elmtyp.o -ifort -c -traceback -O3 ecoute.f90 -o ecoute.o -ifort -c -traceback -O3 runend.f90 -o runend.o -ifort -c -traceback -O3 connpo.f90 -o connpo.o -ifort -c -traceback -O3 vu_msh.f90 -o vu_msh.o -ifort -c -traceback -O3 vu_res.f90 -o vu_res.o -ifort -c -traceback -O3 vu_filter.f90 -o vu_filter.o -ifort -c -traceback -O3 ensmsh.f90 -o ensmsh.o -ifort -c -traceback -O3 ensmsh_bin.f90 -o ensmsh_bin.o -ifort -c -traceback -O3 ensres_bin.f90 -o ensres_bin.o -ifort -c -traceback -O3 ensres_filter.f90 -o ensres_filter.o -ifort -c -traceback -O3 ensmsh_filter.f90 -o ensmsh_filter.o -ifort -c -traceback -O3 ensres.f90 -o ensres.o -ifort -c -traceback -O3 txtres.f90 -o txtres.o -ifort -c -traceback -O3 alya2pos.f90 -o alya2pos.o -ifort -c -traceback -O3 wristl.f90 -o wristl.o -ifort -c -traceback -O3 ensmsh_filter.f90 -o ensmsh_filter.o -ifort -c -traceback -O3 reahed.f90 -o reahed.o -ifort -c -traceback -O3 zfemres.f90 -o zfemres.o -ifort -traceback -O3 -o alya2pos.x *.o +gfortran -c -O3 -cpp def_kintyp.f90 -o def_kintyp.o +gfortran -c -O3 def_elmtyp.f90 -o def_elmtyp.o +gfortran -c -O3 def_inpout.f90 -o def_inpout.o +gfortran -c -O3 gidele.f90 -o gidele.o +gfortran -c -O3 gidres_he.f90 -o gidres_he.o +gfortran -c -O3 gidres_gp.f90 -o gidres_gp.o +gfortran -c -O3 elmtyp.f90 -o elmtyp.o +gfortran -c -O3 ecoute.f90 -o ecoute.o +gfortran -c -O3 runend.f90 -o runend.o +gfortran -c -O3 connpo.f90 -o connpo.o +gfortran -c -O3 vu_msh.f90 -o vu_msh.o +gfortran -c -O3 vu_res.f90 -o vu_res.o +gfortran -c -O3 vu_filter.f90 -o vu_filter.o +gfortran -c -O3 ensmsh.f90 -o ensmsh.o +gfortran -c -O3 ensmsh_bin.f90 -o ensmsh_bin.o +gfortran -c -O3 ensres_bin.f90 -o ensres_bin.o +gfortran -c -O3 ensres_filter.f90 -o ensres_filter.o +gfortran -c -O3 ensmsh_filter.f90 -o ensmsh_filter.o +gfortran -c -O3 ensres.f90 -o ensres.o +gfortran -c -O3 txtres.f90 -o txtres.o +gfortran -c -O3 alya2pos.f90 -o alya2pos.o +gfortran -c -O3 wristl.f90 -o wristl.o +gfortran -c -O3 ensmsh_filter.f90 -o ensmsh_filter.o +gfortran -c -O3 reahed.f90 -o reahed.o +gfortran -c -O3 zfemres.f90 -o zfemres.o +gfortran -O3 -o alya2pos.x *.o rm -rf *.o rm *_genmod.f90 *.mod #+end_example *** Stability check **** Initial rank-elements.dat #+begin_src shell :results output for i in $(seq 1 64); do echo $i 1 done > rank-elements.dat scp rank-elements.dat lille.g5k:. #+end_src #+RESULTS: **** Execution script I have manually ran the script that control the experiment: - Disabling HT - Disabling Turboboost - Log the state of the machines in per-machine ORG files #+begin_src shell :results output source ~/spack-ALYA/share/spack/setup-env.sh export PATH=$(spack location -i openmpi)/bin:$PATH pushd $HOME/WORK-RICARD/resp_sfc/ export EXPEDIR=$HOME/exp_28_chetemi_15_manual/ mkdir -p $EXPEDIR for RUN in $(seq 1 5); do RUNKEY="RUN${RUN}" SCOREPDIR="scorep-${RUNKEY}" echo $RUNKEY # copy machine-file cp $HOME/machine-file . # Run the program $(which mpirun) \ --mca btl_base_warn_component_unused 0 \ --bind-to core:overload-allowed \ --report-bindings \ -x SCOREP_TOTAL_MEMORY=3900MB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np 65 \ -machinefile ./machine-file \ $HOME/alya-bsc-sfc/Executables/unix/Alya.x fensap # Save the data into the SCOREPDIR cp machine-file $SCOREPDIR cp *.log fensap.dat $SCOREPDIR mv domain-*.csv iterations-*.csv $SCOREPDIR cp rank-elements.dat $SCOREPDIR # Move to EXPEDIR mv $SCOREPDIR $EXPEDIR done > execution.log mv execution.log $EXPEDIR popd #+end_src #+RESULTS: : RUN1 : RUN2 : RUN3 : RUN4 : RUN5 **** Data Transformation ***** 1. Read Iteration Timings #+name: exp28_iteration_timings #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; #+end_src #+RESULTS: exp28_iteration_timings #+begin_example Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) Parsed with column specification: cols( X1 = col_double(), X2 = col_double(), X3 = col_double(), X4 = col_integer(), X5 = col_integer() ) #+end_example ***** 2. Transform profile.cubex #+name: exp28_current_dir #+begin_src R :results verbatim :session :exports both print(DIR) #+end_src #+RESULTS: exp28_current_dir : exp_28_chetemi_15_manual//scorep-RUN1 #+name: exp28_cubex_to_open #+header: :var DIR=exp28_current_dir #+begin_src shell :results output cd $DIR ~/install/cube-4.3.5/bin/cube_dump -c all -m all -s csv2 profile.cubex > profile.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex | tail -n+33 | head -n279 | tr '(' ';' | sed -e "s/[[:space:]]*//g" -e "s/,.*$//" -e "s/id=//" -e "s/:[^;]*;/;/" > regions-codes.csv ~/install/cube-4.3.5/bin/cube_dump -w profile.cubex > cube_info.txt #+end_src #+RESULTS: exp28_cubex_to_open ***** 3. Parse the call tree #+name: exp28_cube_calltree #+header: :var dep0=exp28_cubex_to_open #+header: :var DIR=exp28_current_dir #+begin_src perl :results output :exports both use strict; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print $filename_out; #+end_src #+RESULTS: exp28_cube_calltree : exp_28_chetemi_15_manual//scorep-RUN1/cube_info.csv ***** 4. Enrich the call tree #+name: exp28_enrich #+header: :var CSV=exp28_cube_calltree #+header: :var DIR=exp28_current_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); PROFILE = paste0(DIR, "/profile.csv"); REGION = paste0(DIR, "/regions-codes.csv"); df.PROF <- read_csv(PROFILE); exp.REGION <- read_delim(REGION, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) #+end_src #+RESULTS: exp28_enrich #+begin_example Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example ***** X. Verify the load imbalance (and eventually summarize it if necessary) ****** Rough initial LB Analysis #+header: :var dep0=exp28_enrich #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session exp.ENRICH %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-18310RUI/figure18310MBK.png]] ****** Calculate the average #+name: exp28_lb_average #+header: :var dep0=exp28_enrich #+begin_src R :results output :session :exports both exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH exp.ENRICH #+end_src #+RESULTS: exp28_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example ***** 5. The number of elements on T1 #+name: exp28_number_of_elements_shell #+header: :var DIR=exp28_current_dir #+begin_src shell :results table :colnames yes cd $DIR echo "Type Rank NELEM NPOIN NBOUN NPOI32" cat domain-*.csv | grep ^.*T1 #+end_src #+RESULTS: exp28_number_of_elements_shell | Type | Rank | NELEM | NPOIN | NBOUN | NPOI32 | | T1 | 0 | 0 | 0 | 553075 | -1 | | T1 | 1 | 143862 | 27303 | 2832 | 2362 | | T1 | 2 | 143921 | 48988 | 11415 | 4174 | | T1 | 3 | 143822 | 47644 | 9906 | 4947 | | T1 | 4 | 143695 | 43667 | 8197 | 3921 | | T1 | 5 | 143675 | 46242 | 9442 | 4191 | | T1 | 6 | 143752 | 43209 | 7433 | 4876 | | T1 | 7 | 143768 | 48927 | 11007 | 4580 | | T1 | 8 | 143792 | 47241 | 10828 | 2428 | | T1 | 9 | 143945 | 39022 | 6023 | 3238 | | T1 | 10 | 144243 | 47255 | 9907 | 4088 | | T1 | 11 | 143833 | 52178 | 12835 | 4720 | | T1 | 12 | 143842 | 48489 | 13095 | 2082 | | T1 | 13 | 143754 | 47833 | 10503 | 3770 | | T1 | 14 | 143817 | 35239 | 3680 | 3938 | | T1 | 15 | 143776 | 38633 | 5338 | 4204 | | T1 | 16 | 144023 | 37650 | 5066 | 3630 | | T1 | 17 | 143773 | 42217 | 7839 | 3269 | | T1 | 18 | 143746 | 32849 | 2614 | 3662 | | T1 | 19 | 143824 | 37433 | 4732 | 4118 | | T1 | 20 | 143821 | 38632 | 5583 | 3796 | | T1 | 21 | 143849 | 42960 | 8158 | 3320 | | T1 | 22 | 143789 | 46616 | 9577 | 4177 | | T1 | 23 | 143887 | 43733 | 8093 | 3975 | | T1 | 24 | 143817 | 40890 | 6690 | 3904 | | T1 | 25 | 143742 | 37534 | 4465 | 4698 | | T1 | 26 | 143741 | 34257 | 3128 | 3959 | | T1 | 27 | 143887 | 39461 | 6142 | 3517 | | T1 | 28 | 143876 | 35937 | 4111 | 3778 | | T1 | 29 | 144122 | 46780 | 9805 | 3777 | | T1 | 30 | 143919 | 47984 | 11611 | 2013 | | T1 | 31 | 143703 | 43909 | 9221 | 3191 | | T1 | 32 | 143864 | 41551 | 7138 | 3673 | | T1 | 33 | 143920 | 45680 | 9560 | 3251 | | T1 | 34 | 143917 | 43972 | 8473 | 3576 | | T1 | 35 | 143733 | 36957 | 5195 | 2808 | | T1 | 36 | 143974 | 37075 | 5274 | 2810 | | T1 | 37 | 143763 | 33188 | 2855 | 3421 | | T1 | 38 | 143915 | 43860 | 8409 | 3526 | | T1 | 39 | 143733 | 43199 | 7629 | 4442 | | T1 | 40 | 144072 | 50989 | 12285 | 3312 | | T1 | 41 | 143873 | 34536 | 7704 | 3727 | | T1 | 42 | 143719 | 36628 | 6242 | 3728 | | T1 | 43 | 143858 | 46666 | 11830 | 3627 | | T1 | 44 | 143779 | 49048 | 12550 | 4756 | | T1 | 45 | 143927 | 50855 | 11622 | 4467 | | T1 | 46 | 143677 | 51004 | 11178 | 4918 | | T1 | 47 | 143977 | 51247 | 11323 | 4801 | | T1 | 48 | 143872 | 49662 | 11307 | 3910 | | T1 | 49 | 144004 | 43839 | 8154 | 3973 | | T1 | 50 | 143728 | 43759 | 8449 | 3406 | | T1 | 51 | 143841 | 41709 | 7511 | 3472 | | T1 | 52 | 143929 | 50158 | 11469 | 4245 | | T1 | 53 | 143971 | 41111 | 6898 | 3765 | | T1 | 54 | 143831 | 55000 | 13607 | 4988 | | T1 | 55 | 143580 | 55935 | 14390 | 4804 | | T1 | 56 | 143751 | 56151 | 13884 | 6112 | | T1 | 57 | 143993 | 53463 | 13028 | 4431 | | T1 | 58 | 144026 | 55324 | 14121 | 4205 | | T1 | 59 | 143762 | 46085 | 12560 | 4506 | | T1 | 60 | 143832 | 50092 | 12743 | 4424 | | T1 | 61 | 143674 | 44626 | 8912 | 3867 | | T1 | 62 | 143894 | 26865 | 1694 | 2141 | | T1 | 63 | 143735 | 36132 | 7190 | 4182 | | T1 | 64 | 144024 | 38547 | 8615 | 3754 | #+name: exp28_number_of_elements #+header: :var TABLE=exp28_number_of_elements_shell #+begin_src R :results output :session :exports both :colnames yes TABLE %>% as_tibble() %>% filter(Rank != 0) %>% select(-Type) -> exp.ELEMENTS; exp.ELEMENTS; #+end_src #+RESULTS: exp28_number_of_elements #+begin_example # A tibble: 64 x 5 Rank NELEM NPOIN NBOUN NPOI32 1 1 143862 27303 2832 2362 2 2 143921 48988 11415 4174 3 3 143822 47644 9906 4947 4 4 143695 43667 8197 3921 5 5 143675 46242 9442 4191 6 6 143752 43209 7433 4876 7 7 143768 48927 11007 4580 8 8 143792 47241 10828 2428 9 9 143945 39022 6023 3238 10 10 144243 47255 9907 4088 # ... with 54 more rows #+end_example ***** 6. Integrate number of elements with compute cost #+name: exp28_integrate #+header: :var dep1=exp28_lb_average #+header: :var dep0=exp28_number_of_elements #+begin_src R :results output :session :exports both exp.ENRICH %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; df.INTEGRATED #+end_src #+RESULTS: exp28_integrate #+begin_example Joining, by = "Rank" # A tibble: 64 x 3 Rank time NELEM 1 1 28.18224 143862 2 2 41.61312 143921 3 3 40.45403 143822 4 4 38.41553 143695 5 5 40.30081 143675 6 6 37.42849 143752 7 7 42.15025 143768 8 8 42.18351 143792 9 9 36.24237 143945 10 10 40.10288 144243 # ... with 54 more rows #+end_example ***** 7. Do the cumsum #+name: exp28_cumsum #+header: :var dep0=exp28_integrate #+begin_src R :results output :session :exports both df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; df.CUMSUM -> df.exp28.CUMSUM; df.CUMSUM #+end_src #+RESULTS: exp28_cumsum #+begin_example # A tibble: 64 x 5 Rank time NELEM TIMESUM NELEMSUM 1 1 60.43743 184387 0.01510068 0.02002865 2 2 68.98525 135390 0.03233709 0.03473510 3 3 62.07687 134635 0.04784740 0.04935954 4 4 61.41787 140751 0.06319305 0.06464832 5 5 66.77389 137213 0.07987694 0.07955279 6 6 60.27102 146712 0.09493604 0.09548907 7 7 69.88611 141375 0.11239754 0.11084562 8 8 63.52919 131541 0.12827072 0.12513399 9 9 64.41341 152492 0.14436482 0.14169811 10 10 66.84620 136200 0.16106678 0.15649254 # ... with 54 more rows #+end_example ***** 8. Target with Inverting Hilbert Load Curve ****** The function #+name: exp28_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp28_hilbert_invertion_function ****** Calculate the Target #+name: exp28_target #+header: :var dep0=exp28_cumsum #+header: :var dep1=exp28_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp28_tidying #+header: :var dep0=exp28_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ****** Plot with the arrows #+header: :var dep0=exp28_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900T5J.png]] ***** 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp28_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: ***** 9.1 Check which rank has the larger correction #+header: :var dep0=exp28_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.9131876 : 2 62 1.1703797 : 3 63 1.0715776 : 4 64 1.0107468 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. : 32.55 1st Qu.:16.75 1st Qu.: 59.81 Median :32.50 Median : 61.14 Mean :32.50 Mean : 61.80 3rd Qu.:48.25 3rd Qu.: 63.00 Max. :64.00 Max. :103.14 [1] 12.66165 # A tibble: 64 x 2 Rank time 1 31 103.14403 2 59 97.99925 3 46 96.40160 4 44 91.13452 5 3 81.42977 6 20 77.99168 7 1 67.54315 8 36 66.69546 9 56 65.95448 10 39 65.71467 # ... with 54 more rows #+end_example **** Code to get the four runs together in the same data frame Run this once #+begin_src R :results output :session :exports both exp28 = list() #+end_src #+RESULTS: Get one of the following list at each time #+begin_src R :results output :session :exports both DIR=list.files("exp_28_chetemi_15_manual", pattern="^scorep", full.names=TRUE)[[5]] DIR #+end_src #+RESULTS: : [1] "exp_28_chetemi_15_manual/scorep-RUN5" For every time you defined DIR above, run the code below #+header: :var dep0=exp28_enrich #+header: :var dep1=exp28_number_of_elements #+begin_src R :results output :session :exports both DIR position = length(exp28) + 1 exp28[position] <- list(exp.ENRICH %>% left_join(exp.ELEMENTS) %>% mutate(Case = position)); length(exp28); #+end_src #+RESULTS: : [1] "exp_28_chetemi_15_manual/scorep-RUN5" : Joining, by = "Rank" : [1] 5 #+begin_src R :results output :session :exports both exp.ALL <- do.call("bind_rows", exp28) %>% mutate(Case = as.factor(Case)) exp.ALL #+end_src #+RESULTS: #+begin_example # A tibble: 64,000 x 21 Phase Code ID Rank visits time min_time max_time bytes_put 1 1 Computation 14 1 1 28.73742 50.40382 50.40382 0 2 1 Computation 14 2 1 43.04585 50.40640 50.40640 0 3 1 Computation 14 3 1 41.43878 50.40730 50.40730 0 4 1 Computation 14 4 1 38.93097 50.40588 50.40588 0 5 1 Computation 14 5 1 41.11821 50.40721 50.40721 0 6 1 Computation 14 6 1 38.17838 50.40571 50.40571 0 7 1 Computation 14 7 1 42.57766 50.40739 50.40739 0 8 1 Computation 14 8 1 43.47140 50.40639 50.40639 0 9 1 Computation 14 9 1 37.08179 50.40619 50.40619 0 10 1 Computation 14 10 1 40.99082 50.40620 50.40620 0 # ... with 63,990 more rows, and 12 more variables: bytes_get , # ALLOCATION_SIZE , DEALLOCATION_SIZE , bytes_leaked , # maximum_heap_memory_allocated , bytes_sent , # bytes_received , NELEM , NPOIN , NBOUN , NPOI32 , # Case #+end_example **** Textual check just to be sure #+begin_src R :results output :session :exports both exp.ALL %>% filter(Rank == 27) %>% filter(Phase == 2) %>% pull(time) %>% unique #+end_src #+RESULTS: : [1] 36.85080 36.88063 36.85116 36.73310 36.63673 **** Save the data :ATTACH: :PROPERTIES: :Attachments: exp28_stability_check.csv.gz :ID: 05da4262-5e83-43d7-809a-f0f03cd159f2 :END: #+begin_src R :results output :session :exports both write_csv(exp.ALL, "exp28_stability_check.csv.gz") #+end_src #+RESULTS: **** Visualize ***** All points (each facet is a run) #+begin_src R :results output graphics :file img/exp28_stability_check.png :exports both :width 1000 :height 800 :session read_csv("data/05/da4262-5e83-43d7-809a-f0f03cd159f2/exp28_stability_check.csv.gz") %>% group_by(Phase, Rank, Code, Case) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_point() + geom_line(aes(group=Rank), alpha=.2) + ylim(0,NA) + theme_bw(base_size=18) + facet_wrap(~Case) #+end_src #+RESULTS: [[file:img/exp28_stability_check.png]] ***** Per-rank average of points (all iterations of all five runs together) #+begin_src R :results output graphics :file img/exp28_stability_check_v2_phase1.png :exports both :width 1000 :height 800 :session read_csv("data/05/da4262-5e83-43d7-809a-f0f03cd159f2/exp28_stability_check.csv.gz") %>% filter(Phase == 1) %>% group_by(Phase, Rank, Code) %>% summarize(Sum = mean(time), SE=3*sd(time)/sqrt(n())) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=Rank, y=Sum)) + geom_point(alpha=.5) + geom_line(aes(group=Rank), alpha=.2) + geom_errorbar(aes(ymin=Sum-SE, ymax=Sum+SE)) + ylab("Mean") + theme_bw(base_size=18) + facet_wrap(~Phase) #+end_src #+RESULTS: [[file:img/exp28_stability_check_v2_phase1.png]] ***** Per-iteration variability (considering five runs) #+begin_src R :results output graphics :file img/exp28_stability_check_v2.png :exports both :width 1000 :height 1600 :session read_csv("data/05/da4262-5e83-43d7-809a-f0f03cd159f2/exp28_stability_check.csv.gz") %>% group_by(Phase, Rank, Code) %>% summarize(Sum = mean(time), SE=3*sd(time)/sqrt(n())) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=Rank, y=Sum)) + geom_point(alpha=.2) + geom_errorbar(aes(ymin=Sum-SE, ymax=Sum+SE), width=.5) + ylab("Mean") + theme_bw(base_size=18) + facet_wrap(~Phase) #+end_src #+RESULTS: [[file:img/exp28_stability_check_v2.png]] *** Automatize the refinement The workflow has been separated on its own ORG file: - [[./Refinement_Workflow.org]] Check its first section for details on how to use it. ** 23-node chetemi/chifflet (524 cores), 1 it., check automation :EXP29: *** Goal - Verify if automatization works *** Execution script I have manually ran the script that control the experiment: - Disabling HT - Disabling Turboboost - Log the state of the machines in per-machine ORG files #+begin_src shell :results output # Configuration part export STEPS=50 export NP=129 export EXPEKEY=exp_29_chetemi_chifflet_23_manual_${NP} export EXPEDIR=${HOME}/${EXPEKEY} export MACHINEFILE=$HOME/machine-file export CASEDIR=$HOME/WORK-RICARD/resp_sfc/ export CASENAME=fensap export ALYA=$HOME/alya-bsc-sfc/Executables/unix/Alya.x rm -rf $EXPEDIR mkdir -p $EXPEDIR export GITREPO=$HOME/Alya-Perf/ # Control part cat $MACHINEFILE | uniq > /tmp/mf-uniq $GITREPO/scripts/control_experiment.sh /tmp/mf-uniq $EXPEDIR # Use spack source ~/spack-ALYA/share/spack/setup-env.sh # Use the correct mpicc/mpif90/mpirun export PATH=$(spack location -i openmpi)/bin:$PATH # Use the cube_dump for the refinement workflow export PATH=$(spack location -i cube)/bin:$PATH # Generate initial rank-elements.dat to case for i in $(seq 1 $(echo ${NP} - 1 | bc)); do echo $i 1 done > $CASEDIR/rank-elements.dat pushd $CASEDIR for RUN in $(seq 1 ${STEPS}); do RUNKEY="RUN${RUN}_of_${STEPS}" SCOREPDIR="scorep-${RUNKEY}" rm -rf $SCOREPDIR echo $RUNKEY # copy machine-file cp $MACHINEFILE . # Run the program $(which mpirun) \ --mca btl_base_warn_component_unused 0 \ --bind-to core:overload-allowed \ --report-bindings \ -x SCOREP_TOTAL_MEMORY=3900MB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np ${NP} \ -machinefile $MACHINEFILE \ $ALYA $CASENAME # Save the data into the SCOREPDIR cp $MACHINEFILE $SCOREPDIR cp *.log ${CASENAME}.dat $SCOREPDIR mv domain-*.csv iterations-*.csv $SCOREPDIR cp rank-elements.dat $SCOREPDIR # Run the refinement workflow pushd $SCOREPDIR cp $GITREPO/Refinement_Workflow.org . emacs -batch -l ~/.emacs.d/init.el \ --eval "(setq vc-follow-symlinks nil)" \ --eval "(setq ess-ask-for-ess-directory nil)" \ --eval "(setq org-export-babel-evaluate t)" \ Refinement_Workflow.org \ --funcall org-babel-execute-buffer # Adjust rank-elements_next.dat NRANKS=rank-elements_next.dat NRANKSNEW=rank-elements_next_new.dat MAXRANK=$(cat $NRANKS | tail -n1 | cut -d" " -f1) MAXRANKVALUE=$(cat $NRANKS | tail -n1 | cut -d" " -f2) SUMSEC=$(echo $(cat $NRANKS | cut -d" " -f2 | tr '\n' '+' | sed 's/+$//') | bc -l) DIFF=$(echo "$MAXRANK - $SUMSEC" | bc -l) head -n-1 $NRANKS > $NRANKSNEW echo "$MAXRANK $(echo "$MAXRANKVALUE + $DIFF" | bc -l)" >> $NRANKSNEW cp $NRANKSNEW $CASEDIR/rank-elements.dat popd # Move all data to EXPEDIR for archiving purposes mv $SCOREPDIR $EXPEDIR done > execution.log mv execution.log $EXPEDIR popd #+end_src *** Analysis - Program has executed for 13 times before crashing ** 27-nodes (432 cores) parasilo, 1 it., check update :FAIL:EXP30: *** Goal - Verify if 256+1 works now with the update at Alya - Launch from =parasilo-1=. *** Execution script #+begin_src shell :results output # Configuration part export STEPS=3 export NP=257 export EXPEKEY=exp_30_parasilo_27_manual_${NP} export EXPEDIR=${HOME}/${EXPEKEY} export MACHINEFILE=$HOME/machine-file export CASEDIR=$HOME/WORK-RICARD/resp_sfc/ export CASENAME=fensap export ALYA=$HOME/svn-bsc-sfc-2/Executables/unix/Alya.x rm -rf $EXPEDIR mkdir -p $EXPEDIR export GITREPO=$HOME/Alya-Perf/ # Control part cat $MACHINEFILE | uniq > /tmp/mf-uniq $GITREPO/scripts/control_experiment.sh /tmp/mf-uniq $EXPEDIR # Use spack source ~/spack-ALYA/share/spack/setup-env.sh # Use the correct mpicc/mpif90/mpirun export PATH=$(spack location -i openmpi)/bin:$PATH # Use the cube_dump for the refinement workflow export PATH=$(spack location -i cube)/bin:$PATH # Generate initial rank-elements.dat to case for i in $(seq 1 $(echo ${NP} - 1 | bc)); do echo $i 1 done > $CASEDIR/rank-elements.dat pushd $CASEDIR for RUN in $(seq 1 ${STEPS}); do RUNKEY="RUN${RUN}_of_${STEPS}" SCOREPDIR="scorep-${RUNKEY}" rm -rf $SCOREPDIR echo $RUNKEY # copy machine-file cp $MACHINEFILE . # Run the program $(which mpirun) \ --mca btl_base_warn_component_unused 0 \ --bind-to core:overload-allowed \ --report-bindings \ -x SCOREP_TOTAL_MEMORY=3900MB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np ${NP} \ -machinefile $MACHINEFILE \ $ALYA $CASENAME # Save the data into the SCOREPDIR cp $MACHINEFILE $SCOREPDIR cp *.log ${CASENAME}.dat $SCOREPDIR mv domain-*.csv iterations-*.csv $SCOREPDIR cp rank-elements.dat $SCOREPDIR # Run the refinement workflow pushd $SCOREPDIR cp $GITREPO/Refinement_Workflow.org . emacs -batch -l ~/.emacs.d/init.el \ --eval "(setq vc-follow-symlinks nil)" \ --eval "(setq ess-ask-for-ess-directory nil)" \ --eval "(setq org-export-babel-evaluate t)" \ Refinement_Workflow.org \ --funcall org-babel-execute-buffer # Adjust rank-elements_next.dat NRANKS=rank-elements_next.dat NRANKSNEW=rank-elements_next_new.dat MAXRANK=$(cat $NRANKS | tail -n1 | cut -d" " -f1) MAXRANKVALUE=$(cat $NRANKS | tail -n1 | cut -d" " -f2) SUMSEC=$(echo $(cat $NRANKS | cut -d" " -f2 | tr '\n' '+' | sed 's/+$//') | bc -l) DIFF=$(echo "$MAXRANK - $SUMSEC" | bc -l) head -n-1 $NRANKS > $NRANKSNEW echo "$MAXRANK $(echo "$MAXRANKVALUE + $DIFF" | bc -l)" >> $NRANKSNEW cp $NRANKSNEW $CASEDIR/rank-elements.dat popd # Move all data to EXPEDIR for archiving purposes mv $SCOREPDIR $EXPEDIR done > execution.log mv execution.log $EXPEDIR popd #+end_src *** Summary - Program Alya didn't work. ** 4-node (76 cores) rennes, check update :EXP31: *** Goal - Verify if Alya works - Pre-check - Install xclip ess *** Execution script #+begin_src shell :results output # Configuration part export STEPS=5 export NSTEPS=1 export NP=76 export EXPEKEY=exp_31_rennes_4_manual_${NP} export EXPEDIR=${HOME}/${EXPEKEY} export MACHINEFILE=$HOME/machine-file export CASEDIR=$HOME/WORK-RICARD/resp_sfc/ export CASENAME=fensap export ALYA=$HOME/alya-bsc-sfc/Executables/unix/Alya.g rm -rf $EXPEDIR mkdir -p $EXPEDIR export GITREPO=$HOME/Alya-Perf/ # Control part cat $MACHINEFILE | uniq > /tmp/mf-uniq $GITREPO/scripts/control_experiment.sh /tmp/mf-uniq $EXPEDIR # Use spack source ~/spack-ALYA/share/spack/setup-env.sh # Use the correct mpicc/mpif90/mpirun export PATH=$(spack location -i openmpi)/bin:$PATH # Use the cube_dump for the refinement workflow export PATH=$(spack location -i cube)/bin:$PATH # Generate initial rank-elements.dat to case for i in $(seq 1 $(echo ${NP} - 1 | bc)); do echo $i 1 done > $CASEDIR/rank-elements.dat pushd $CASEDIR sed -i "s/NUMBER_OF_STEPS=.*$/NUMBER_OF_STEPS=${NSTEPS}/" fensap.dat for RUN in $(seq 1 ${STEPS}); do RUNKEY="RUN${RUN}_of_${STEPS}" SCOREPDIR="scorep-${RUNKEY}" rm -rf $SCOREPDIR echo $RUNKEY # Run the program $(which mpirun) \ --mca btl_base_warn_component_unused 0 \ --bind-to core:overload-allowed \ --report-bindings \ -x SCOREP_TOTAL_MEMORY=3900MB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np ${NP} \ -machinefile $MACHINEFILE \ $ALYA $CASENAME # Save the data into the SCOREPDIR cp $MACHINEFILE $SCOREPDIR cp *.log ${CASENAME}.dat $SCOREPDIR mv domain-*.csv iterations-*.csv $SCOREPDIR cp rank-elements.dat $SCOREPDIR # Run the refinement workflow pushd $SCOREPDIR cp $GITREPO/Refinement_Workflow.org . emacs -batch -l ~/.emacs.d/init.el \ --eval "(setq vc-follow-symlinks nil)" \ --eval "(setq ess-ask-for-ess-directory nil)" \ --eval "(setq org-export-babel-evaluate t)" \ Refinement_Workflow.org \ --funcall org-babel-execute-buffer # Adjust rank-elements_next.dat NRANKS=rank-elements_next.dat cp $NRANKS $CASEDIR/rank-elements.dat popd # Move all data to EXPEDIR for archiving purposes mv $SCOREPDIR $EXPEDIR done > execution.log mv execution.log $EXPEDIR popd #+end_src ** 65-nodes (1040 cores) rennes paravance, check update :EXP32: *** Goal - Image has been update with xclip ess - Verify if Alya works up to 1024 cores - paravance-7[0123] with major problems - Unable to launch mpirun because of that - Solution was to remove then from the machinefile *** Execution script #+begin_src shell :results output # Configuration part export STEPS=5 export NSTEPS=10 export EXPEKEY=exp_32_paravance_68_manual export EXPEDIR=${HOME}/${EXPEKEY} export MACHINEFILE=$HOME/paravance export CASEDIR=$HOME/WORK-RICARD/resp_sfc/ export CASENAME=fensap export ALYA=$HOME/alya-bsc-sfc/Executables/unix/Alya.x rm -rf $EXPEDIR mkdir -p $EXPEDIR export GITREPO=$HOME/Alya-Perf/ # Control part cat $MACHINEFILE | uniq > /tmp/mf-uniq $GITREPO/scripts/control_experiment.sh /tmp/mf-uniq $EXPEDIR # Use spack source ~/spack-ALYA/share/spack/setup-env.sh # Use the correct mpicc/mpif90/mpirun export PATH=$(spack location -i openmpi)/bin:$PATH # Use the cube_dump for the refinement workflow export PATH=$(spack location -i cube)/bin:$PATH pushd $CASEDIR sed -i "s/NUMBER_OF_STEPS=.*$/NUMBER_OF_STEPS=${NSTEPS}/" fensap.dat for NP in 1025 513 257 129; do # Generate initial rank-elements.dat to case for i in $(seq 1 $(echo ${NP} - 1 | bc)); do echo $i 1 done > $CASEDIR/rank-elements.dat for RUN in $(seq 1 ${STEPS}); do RUNKEY="RUN_${NP}_STEP_${RUN}_of_${STEPS}" SCOREPDIR="scorep-${RUNKEY}" rm -rf $SCOREPDIR echo $RUNKEY # Run the program $(which mpirun) \ --mca btl_base_warn_component_unused 0 \ --bind-to core:overload-allowed \ --report-bindings \ -x SCOREP_TOTAL_MEMORY=3900MB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np ${NP} \ -machinefile $MACHINEFILE \ $ALYA $CASENAME # Save the data into the SCOREPDIR cp $MACHINEFILE $SCOREPDIR cp *.log ${CASENAME}.dat $SCOREPDIR mv domain-*.csv iterations-*.csv $SCOREPDIR cp rank-elements.dat $SCOREPDIR # Run the refinement workflow pushd $SCOREPDIR cp $GITREPO/Refinement_Workflow.org . emacs -batch -l ~/.emacs.d/init.el \ --eval "(setq vc-follow-symlinks nil)" \ --eval "(setq ess-ask-for-ess-directory nil)" \ --eval "(setq org-export-babel-evaluate t)" \ Refinement_Workflow.org \ --funcall org-babel-execute-buffer # Adjust rank-elements_next.dat NRANKS=rank-elements_next.dat cp $NRANKS $CASEDIR/rank-elements.dat popd # Move all data to EXPEDIR for archiving purposes mv $SCOREPDIR $EXPEDIR done done > execution.log mv execution.log $EXPEDIR popd #+end_src *** Data Transformation **** 3. Parse the call tree (code block has been tangled and committed) #+begin_src perl :results output :exports both :tangle scripts/cube_calltree.pl :tangle-mode (identity #o755) #!/usr/bin/perl use strict; my($DIR) = $ARGV[0]; my($filename) = $DIR . "/cube_info.txt"; my($line); open(INPUT,$filename); my($in_CALLTREE) = 0; my($VAR_iteration) = -1; my($VAR_type) = -1; my($VAR_id) = -1; my($filename_out) = $filename; $filename_out =~ s/txt$/csv/; open(OUTPUT,"> ".$filename_out); while(defined($line=)) { chomp $line; if($line =~ "CALL TREE") { $in_CALLTREE = 1; } if(!$in_CALLTREE) { next; } if($line =~ "SYSTEM DIMENSION") { $in_CALLTREE = 0; } if($line =~ /^iter(\d*)\s*.*id=(\d*),.*/) { $VAR_iteration = $1; $VAR_type = "Computation"; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } if($line =~ /^\s*\|\-(\S*)\s.*id=(\d*),.*/) { # print $line."\n"; $VAR_type = $1; $VAR_id = $2; # print "|$VAR_iteration | $VAR_type | $VAR_id |\n"; print OUTPUT "$VAR_iteration,$VAR_type,$VAR_id\n"; } } close(OUTPUT); print($filename_out . "\n"); #+end_src #+RESULTS: exp32_cube_calltree : exp_20_grisou_4_manual/scorep-20180125_1159_34483171676480//cube_info.csv **** 5. The number of elements on T1 (code block has been tangled and committed) #+begin_src shell :results output :tangle scripts/no_elements_domain.sh :tangle-mode (identity #o755) #!/bin/bash DIR=$1 FILE=$2 pushd $DIR echo "Rank,NELEM,NPOIN,NBOUN,NPOI32" > $FILE zcat domain-*.csv.gz | grep ^.*T1 | sed -e "s/ T1//" -e "s/[[:space:]]\+/ /g" -e "s/^ //" -e "s/ /,/g" >> $FILE popd #+end_src **** 0. DIR #+begin_src R :results output :session :exports both DIR="exp_32_paravance_65_manual/scorep-RUN_513_STEP_2_of_5" #+end_src #+RESULTS: #+name: exp32_current_dir #+begin_src R :results verbatim :session :exports both print(DIR) #+end_src #+RESULTS: exp32_current_dir : exp_32_paravance_65_manual/scorep-RUN_513_STEP_2_of_5 **** Read one DIR function #+name: exp32_read_one_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); read_one_DIR <- function(DIR) { # Get basic info about the experience tibble(Dir=DIR) %>% separate(Dir, into=c("X", "Case"), sep="/") %>% select(-X) %>% separate(Case, into=c("X1", "NP", "X3", "Seq", "X5", "X6"), sep="_") %>% select(-X1, -X3, -X5, -X6) -> basic.info; # 2. Transform profile.cubex PROFILE.CUBEX = paste0(DIR, "/profile.cubex"); PROFILE.CSV = paste0(DIR, "/profile.csv"); REGIONSCODES.CSV = paste0(DIR, "/regions-codes.csv"); CUBEINFO.TXT = paste0(DIR, "/cube_info.txt"); CUBEINFO.CSV = paste0(DIR, "/cube_info.csv"); system2("cube_dump", args=paste("-c all -m all -s csv2", PROFILE.CUBEX, ">", PROFILE.CSV)); system2("cube_dump", args=paste("-w", PROFILE.CUBEX, "| tail -n+33 | head -n279 | tr '(' ';' | sed -e \"s/[[:space:]]*//g\" -e \"s/,.*$//\" -e \"s/id=//\" -e \"s/:[^;]*;/;/\" > ", REGIONSCODES.CSV)); system2("cube_dump", args=paste("-w", PROFILE.CUBEX, ">", CUBEINFO.TXT)); system2("./scripts/cube_calltree.pl", args=DIR); # 1. Read Iteration Timings if(FALSE){ do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv.gz$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; } # 4. Enrich the call tree df.PROF <- read_csv(PROFILE.CSV); exp.REGION <- read_delim(REGIONSCODES.CSV, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CUBEINFO.CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) %>% mutate(NP = basic.info$NP, Seq = basic.info$Seq) return(exp.ENRICH); # 5. The number of elements on T1 NOELEMENTS.CSV = "no_elements.csv"; system2("./scripts/no_elements_domain.sh", args=paste(DIR, NOELEMENTS.CSV)); exp.ELEMENTS <- read_csv(paste0(DIR, "/", NOELEMENTS.CSV)) %>% filter(Rank != 0) # Calculate the average exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH.AVERAGE; # 6. Integrate number of elements with compute cost exp.ENRICH.AVERAGE %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; # 7. Do the cumsum df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; } #+end_src #+RESULTS: exp32_read_one_dir **** Test read one DIR #+header: :var dep0=exp32_read_one_dir #+begin_src R :results output :session :exports both df.TMP <- read_one_DIR(DIR) #+end_src #+RESULTS: #+begin_example exp_32_paravance_65_manual/scorep-RUN_513_STEP_2_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) Warning message: In rbind(names(probs), probs_f) : number of columns of result is not a multiple of vector length (arg 1) #+end_example #+begin_src R :results output :session :exports both df.TMP #+end_src #+RESULTS: #+begin_example # A tibble: 5,120 x 18 Phase Code ID Rank visits time min_time max_time bytes_put 1 1 Computation 14 1 1 4.485452 10.63609 10.63609 0 2 1 Computation 14 2 1 4.495881 10.63586 10.63586 0 3 1 Computation 14 3 1 4.677876 10.63597 10.63597 0 4 1 Computation 14 4 1 4.510341 10.63589 10.63589 0 5 1 Computation 14 5 1 4.474415 10.63602 10.63602 0 6 1 Computation 14 6 1 4.673044 10.63584 10.63584 0 7 1 Computation 14 7 1 4.433373 10.63605 10.63605 0 8 1 Computation 14 8 1 4.791339 10.63593 10.63593 0 9 1 Computation 14 9 1 4.694905 10.63616 10.63616 0 10 1 Computation 14 10 1 4.711255 10.63598 10.63598 0 # ... with 5,110 more rows, and 9 more variables: bytes_get , # ALLOCATION_SIZE , DEALLOCATION_SIZE , bytes_leaked , # maximum_heap_memory_allocated , bytes_sent , # bytes_received , NP , Seq #+end_example **** Read ALL DIRs #+begin_src R :results output :session :exports both do.call("bind_rows", lapply(list.files("exp_32_paravance_65_manual", pattern="scorep", full.names=TRUE), function(case) { read_one_DIR(case); } )) -> exp32.ALL; #+end_src #+RESULTS: #+begin_example exp_32_paravance_65_manual/scorep-RUN_1025_STEP_1_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 20500 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 15376 bytes_sent an integer 12200300300 file 2 15376 bytes_received an integer 12200300300 row 3 15377 bytes_sent an integer 12200300300 col 4 15377 bytes_received an integer 12200300300 expected 5 15378 bytes_sent an integer 12200300300 actual # ... with 1 more variables: file ... ................. ... ............................................. ........ ............................................. ...... ............................................. .... ............................................. ... ............................................. ... ............................................. ........ ............................................. ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_1025_STEP_2_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 20500 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 15376 bytes_sent an integer 12200300300 file 2 15376 bytes_received an integer 12200300300 row 3 15377 bytes_sent an integer 12200300300 col 4 15377 bytes_received an integer 12200300300 expected 5 15378 bytes_sent an integer 12200300300 actual # ... with 1 more variables: file ... ................. ... ............................................. ........ ............................................. ...... ............................................. .... ............................................. ... ............................................. ... ............................................. ........ ............................................. ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_1025_STEP_3_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 20500 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 15376 bytes_sent an integer 12200300300 file 2 15376 bytes_received an integer 12200300300 row 3 15377 bytes_sent an integer 12200300300 col 4 15377 bytes_received an integer 12200300300 expected 5 15378 bytes_sent an integer 12200300300 actual # ... with 1 more variables: file ... ................. ... ............................................. ........ ............................................. ...... ............................................. .... ............................................. ... ............................................. ... ............................................. ........ ............................................. ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_1025_STEP_4_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 20500 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 15376 bytes_sent an integer 12200300300 file 2 15376 bytes_received an integer 12200300300 row 3 15377 bytes_sent an integer 12200300300 col 4 15377 bytes_received an integer 12200300300 expected 5 15378 bytes_sent an integer 12200300300 actual # ... with 1 more variables: file ... ................. ... ............................................. ........ ............................................. ...... ............................................. .... ............................................. ... ............................................. ... ............................................. ........ ............................................. ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_1025_STEP_5_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 20500 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 15376 bytes_sent an integer 12200300300 file 2 15376 bytes_received an integer 12200300300 row 3 15377 bytes_sent an integer 12200300300 col 4 15377 bytes_received an integer 12200300300 expected 5 15378 bytes_sent an integer 12200300300 actual # ... with 1 more variables: file ... ................. ... ............................................. ........ ............................................. ...... ............................................. .... ............................................. ... ............................................. ... ............................................. ........ ............................................. ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_129_STEP_1_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_129_STEP_2_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_129_STEP_3_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_129_STEP_4_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_129_STEP_5_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_257_STEP_1_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 4626 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 3856 bytes_sent an integer 3059002124 file 2 3856 bytes_received an integer 3059002124 row 3 3857 bytes_sent an integer 3059002124 col 4 3857 bytes_received an integer 3059002124 expected 5 3858 bytes_sent an integer 3059002124 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_257_STEP_2_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 4626 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 3856 bytes_sent an integer 3059002124 file 2 3856 bytes_received an integer 3059002124 row 3 3857 bytes_sent an integer 3059002124 col 4 3857 bytes_received an integer 3059002124 expected 5 3858 bytes_sent an integer 3059002124 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_257_STEP_3_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 4626 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 3856 bytes_sent an integer 3059002124 file 2 3856 bytes_received an integer 3059002124 row 3 3857 bytes_sent an integer 3059002124 col 4 3857 bytes_received an integer 3059002124 expected 5 3858 bytes_sent an integer 3059002124 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_257_STEP_4_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 4626 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 3856 bytes_sent an integer 3059002124 file 2 3856 bytes_received an integer 3059002124 row 3 3857 bytes_sent an integer 3059002124 col 4 3857 bytes_received an integer 3059002124 expected 5 3858 bytes_sent an integer 3059002124 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_257_STEP_5_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 4626 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 3856 bytes_sent an integer 3059002124 file 2 3856 bytes_received an integer 3059002124 row 3 3857 bytes_sent an integer 3059002124 col 4 3857 bytes_received an integer 3059002124 expected 5 3858 bytes_sent an integer 3059002124 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_513_STEP_1_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_513_STEP_2_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_513_STEP_3_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) |===== | 7% |====== | 8% |======= | 9% |======= | 10% |======== | 12% |========= | 13% |========== | 14% |=========== | 15% |============ | 16% |============= | 18% |============== | 19% |=============== | 20% |=============== | 21% |=============== | 23% 1 MB |=============== | 24% 1 MB |================ | 25% 1 MB |================= | 26% 1 MB |================== | 28% 1 MB |=================== | 29% 1 MB |=================== | 30% 1 MB |==================== | 31% 1 MB |===================== | 33% 1 MB |====================== | 34% 1 MB |====================== | 35% 1 MB |======================= | 36% 1 MB |======================== | 37% 1 MB |========================= | 39% 1 MB |========================== | 40% 1 MB |=========================== | 41% 1 MB |=========================== | 42% 1 MB |============================ | 43% 1 MB |============================= | 45% 1 MB |============================== | 46% 2 MB |=============================== | 47% 2 MB |=============================== | 48% 2 MB |================================ | 50% 2 MB |================================= | 51% 2 MB |================================== | 52% 2 MB |================================== | 53% 2 MB |=================================== | 55% 2 MB |==================================== | 56% 2 MB |===================================== | 57% 2 MB |====================================== | 58% 2 MB |======================================= | 60% 2 MB |======================================= | 61% 2 MB |======================================== | 62% 2 MB |========================================= | 63% 2 MB |========================================== | 65% 2 MB |=========================================== | 66% 2 MB |============================================ | 67% 2 MB |============================================= | 69% 3 MB |============================================= | 70% 3 MB |============================================== | 71% 3 MB |=============================================== | 73% 3 MB |================================================ | 74% 3 MB |================================================= | 75% 3 MB |================================================== | 77% 3 MB |=================================================== | 78% 3 MB |=================================================== | 79% 3 MB |==================================================== | 81% 3 MB |===================================================== | 82% 3 MB |====================================================== | 83% 3 MB |======================================================= | 85% 3 MB |======================================================== | 86% 3 MB |========================================================= | 87% 3 MB |========================================================== | 89% 3 MB |========================================================== | 90% 3 MB |=========================================================== | 91% 3 MB |============================================================ | 93% 4 MB |============================================================= | 94% 4 MB |============================================================== | 95% 4 MB |=============================================================== | 97% 4 MB |================================================================| 98% 4 MB |================================================================| 99% 4 MB |=================================================================| 100% 4 MB Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_513_STEP_4_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_32_paravance_65_manual/scorep-RUN_513_STEP_5_of_5/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) There were 15 warnings (use warnings() to see them) #+end_example #+begin_src R :results output :session :exports both exp32.ALL #+end_src **** ALL Rough initial LB Analysis #+begin_src R :results output graphics :file img/exp32_ALL.png :exports both :width 1200 :height 800 :session exp32.ALL %>% mutate(NP = as.integer(NP), Seq = as.integer(Seq)) %>% group_by(NP, Seq, Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_line(aes(group=Rank), alpha=.1) + geom_point() + ylim(0,NA) + theme_bw(base_size=18) + facet_grid(NP~Seq, scales="free") #+end_src #+RESULTS: [[file:img/exp32_ALL.png]] **** Save data for Arnaud's analysis :ATTACH: :PROPERTIES: :Attachments: exp32_compute_ALL.csv.gz :ID: 572c142b-dd52-4aff-bc0a-1c26f5bad357 :END: #+begin_src R :results output :session :exports both write_csv(exp32.ALL, "exp32_compute_ALL.csv.gz"); #+end_src #+RESULTS: **** TODO X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session df.TMP %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_line(aes(group=Rank), alpha=.1) + geom_point() + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-22415aoO/figure22415k9p.png]] ***** Calculate the average #+name: exp32_lb_average #+header: :var dep0=exp32_enrich #+begin_src R :results output :session :exports both exp.ENRICH #+end_src #+RESULTS: exp32_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** TODO 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp32_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp32_hilbert_invertion_function ***** Calculate the Target #+name: exp32_target #+header: :var dep0=exp32_cumsum #+header: :var dep1=exp32_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp32_tidying #+header: :var dep0=exp32_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp32_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900T5J.png]] **** TODO 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp32_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** TODO 9.1 Check which rank has the larger correction #+header: :var dep0=exp32_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.9131876 : 2 62 1.1703797 : 3 63 1.0715776 : 4 64 1.0107468 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. : 32.55 1st Qu.:16.75 1st Qu.: 59.81 Median :32.50 Median : 61.14 Mean :32.50 Mean : 61.80 3rd Qu.:48.25 3rd Qu.: 63.00 Max. :64.00 Max. :103.14 [1] 12.66165 # A tibble: 64 x 2 Rank time 1 31 103.14403 2 59 97.99925 3 46 96.40160 4 44 91.13452 5 3 81.42977 6 20 77.99168 7 1 67.54315 8 36 66.69546 9 56 65.95448 10 39 65.71467 # ... with 54 more rows #+end_example ** 65-nodes (1040 cores) rennes paravance, check many rounds :EXP33: *** Execution script #+begin_src shell :results output # Configuration part export STEPS=50 export TIMESTEPS=10 export EXPEKEY=exp_33_paravance_65_manual export EXPEDIR=${HOME}/${EXPEKEY} export MACHINEFILE=$HOME/paravance export CASEDIR=$HOME/WORK-RICARD/resp_sfc/ export CASENAME=fensap export ALYA=$HOME/alya-bsc-sfc/Executables/unix/Alya.x rm -rf $EXPEDIR mkdir -p $EXPEDIR export GITREPO=$HOME/Alya-Perf/ # Control part cat $MACHINEFILE | uniq > /tmp/mf-uniq $GITREPO/scripts/control_experiment.sh /tmp/mf-uniq $EXPEDIR # Use spack source ~/spack-ALYA/share/spack/setup-env.sh # Use the correct mpicc/mpif90/mpirun export PATH=$(spack location -i openmpi)/bin:$PATH # Use the cube_dump for the refinement workflow export PATH=$(spack location -i cube)/bin:$PATH pushd $CASEDIR sed -i "s/NUMBER_OF_STEPS=.*$/NUMBER_OF_STEPS=${TIMESTEPS}/" fensap.dat for NP in 513 1025; do # Generate initial rank-elements.dat to case for i in $(seq 1 $(echo ${NP} - 1 | bc)); do echo $i 1 done > $CASEDIR/rank-elements.dat for RUN in $(seq 1 ${STEPS}); do RUNKEY="RUN_${NP}_STEP_${RUN}_of_${STEPS}" SCOREPDIR="scorep-${RUNKEY}" rm -rf $SCOREPDIR echo $RUNKEY # Run the program $(which mpirun) \ --mca btl_base_warn_component_unused 0 \ --bind-to core:overload-allowed \ --report-bindings \ -x SCOREP_TOTAL_MEMORY=3900MB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np ${NP} \ -machinefile $MACHINEFILE \ $ALYA $CASENAME # Save the data into the SCOREPDIR cp $MACHINEFILE $SCOREPDIR cp *.log ${CASENAME}.dat $SCOREPDIR mv domain-*.csv iterations-*.csv $SCOREPDIR cp rank-elements.dat $SCOREPDIR # Run the refinement workflow pushd $SCOREPDIR cp $GITREPO/Refinement_Workflow.org . emacs -batch -l ~/.emacs.d/init.el \ --eval "(setq vc-follow-symlinks nil)" \ --eval "(setq ess-ask-for-ess-directory nil)" \ --eval "(setq org-export-babel-evaluate t)" \ Refinement_Workflow.org \ --funcall org-babel-execute-buffer # Compress data (after workflow) gzip *.csv # Adjust rank-elements_next.dat NRANKS=rank-elements_next.dat cp $NRANKS $CASEDIR/rank-elements.dat popd # Move all data to EXPEDIR for archiving purposes mv $SCOREPDIR $EXPEDIR done done > execution.log mv execution.log $EXPEDIR popd #+end_src *** Data Transformation **** Read one DIR function #+name: exp33_read_one_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); read_one_DIR <- function(DIR) { # Get basic info about the experience tibble(Dir=DIR) %>% separate(Dir, into=c("X", "Case"), sep="/") %>% select(-X) %>% separate(Case, into=c("X1", "NP", "X3", "Seq", "X5", "X6"), sep="_") %>% select(-X1, -X3, -X5, -X6) -> basic.info; # 2. Transform profile.cubex PROFILE.CUBEX = paste0(DIR, "/profile.cubex"); PROFILE.CSV = paste0(DIR, "/profile.csv"); REGIONSCODES.CSV = paste0(DIR, "/regions-codes.csv"); CUBEINFO.TXT = paste0(DIR, "/cube_info.txt"); CUBEINFO.CSV = paste0(DIR, "/cube_info.csv"); system2("cube_dump", args=paste("-c all -m all -s csv2", PROFILE.CUBEX, ">", PROFILE.CSV)); system2("cube_dump", args=paste("-w", PROFILE.CUBEX, "| tail -n+33 | head -n279 | tr '(' ';' | sed -e \"s/[[:space:]]*//g\" -e \"s/,.*$//\" -e \"s/id=//\" -e \"s/:[^;]*;/;/\" > ", REGIONSCODES.CSV)); system2("cube_dump", args=paste("-w", PROFILE.CUBEX, ">", CUBEINFO.TXT)); system2("./scripts/cube_calltree.pl", args=DIR); # 1. Read Iteration Timings if(FALSE){ do.call("bind_rows", lapply(list.files(DIR, pattern="iterations.*csv.gz$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; } # 4. Enrich the call tree df.PROF <- read_csv(PROFILE.CSV); exp.REGION <- read_delim(REGIONSCODES.CSV, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CUBEINFO.CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) %>% mutate(NP = basic.info$NP, Seq = basic.info$Seq) return(exp.ENRICH); # 5. The number of elements on T1 NOELEMENTS.CSV = "no_elements.csv"; system2("./scripts/no_elements_domain.sh", args=paste(DIR, NOELEMENTS.CSV)); exp.ELEMENTS <- read_csv(paste0(DIR, "/", NOELEMENTS.CSV)) %>% filter(Rank != 0) # Calculate the average exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH.AVERAGE; # 6. Integrate number of elements with compute cost exp.ENRICH.AVERAGE %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; # 7. Do the cumsum df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; } #+end_src #+RESULTS: exp33_read_one_dir **** Read ALL DIRs #+begin_src R :results output :session :exports both do.call("bind_rows", lapply(list.files("exp_33_paravance_65_manual", pattern="scorep", full.names=TRUE), function(case) { read_one_DIR(case); } )) -> exp33.ALL; #+end_src #+RESULTS: #+begin_example exp_33_paravance_65_manual/scorep-RUN_513_STEP_1_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_10_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_11_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_12_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_13_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_14_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_15_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_16_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_17_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_18_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_19_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_2_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_20_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_21_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_22_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_23_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_24_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_25_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_26_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_27_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_28_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_29_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) |=========== | 15% |============ | 16% |============= | 18% |============== | 19% |=============== | 20% |=============== | 21% |=============== | 23% 1 MB |=============== | 24% 1 MB |================ | 25% 1 MB |================= | 26% 1 MB |================== | 28% 1 MB |=================== | 29% 1 MB |=================== | 30% 1 MB |==================== | 31% 1 MB |===================== | 33% 1 MB |====================== | 34% 1 MB |====================== | 35% 1 MB |======================= | 36% 1 MB |======================== | 37% 1 MB |========================= | 39% 1 MB |========================== | 40% 1 MB |=========================== | 41% 1 MB |=========================== | 42% 1 MB |============================ | 43% 1 MB |============================= | 45% 1 MB |============================== | 46% 2 MB |=============================== | 47% 2 MB |=============================== | 48% 2 MB |================================ | 50% 2 MB |================================= | 51% 2 MB |================================== | 52% 2 MB |================================== | 53% 2 MB |=================================== | 55% 2 MB |==================================== | 56% 2 MB |===================================== | 57% 2 MB |====================================== | 58% 2 MB |======================================= | 60% 2 MB |======================================= | 61% 2 MB |======================================== | 62% 2 MB |========================================= | 63% 2 MB |========================================== | 65% 2 MB |=========================================== | 66% 2 MB |============================================ | 67% 2 MB |============================================= | 69% 3 MB |============================================= | 70% 3 MB |============================================== | 71% 3 MB |=============================================== | 73% 3 MB |================================================ | 74% 3 MB |================================================= | 75% 3 MB |================================================== | 77% 3 MB |=================================================== | 78% 3 MB |=================================================== | 79% 3 MB |==================================================== | 81% 3 MB |===================================================== | 82% 3 MB |====================================================== | 83% 3 MB |======================================================= | 85% 3 MB |======================================================== | 86% 3 MB |========================================================= | 87% 3 MB |========================================================== | 89% 3 MB |========================================================== | 90% 3 MB |=========================================================== | 91% 3 MB |============================================================ | 93% 4 MB |============================================================= | 94% 4 MB |============================================================== | 95% 4 MB |=============================================================== | 97% 4 MB |================================================================| 98% 4 MB |================================================================| 99% 4 MB |=================================================================| 100% 4 MB Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_3_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_30_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_31_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_32_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) |============== | 19% |=============== | 20% |=============== | 21% |=============== | 23% 1 MB |=============== | 24% 1 MB |================ | 25% 1 MB |================= | 26% 1 MB |================== | 28% 1 MB |=================== | 29% 1 MB |=================== | 30% 1 MB |==================== | 31% 1 MB |===================== | 33% 1 MB |====================== | 34% 1 MB |====================== | 35% 1 MB |======================= | 36% 1 MB |======================== | 37% 1 MB |========================= | 39% 1 MB |========================== | 40% 1 MB |=========================== | 41% 1 MB |=========================== | 42% 1 MB |============================ | 43% 1 MB |============================= | 45% 1 MB |============================== | 46% 2 MB |=============================== | 47% 2 MB |=============================== | 48% 2 MB |================================ | 50% 2 MB |================================= | 51% 2 MB |================================== | 52% 2 MB |================================== | 53% 2 MB |=================================== | 55% 2 MB |==================================== | 56% 2 MB |===================================== | 57% 2 MB |====================================== | 58% 2 MB |======================================= | 60% 2 MB |======================================= | 61% 2 MB |======================================== | 62% 2 MB |========================================= | 63% 2 MB |========================================== | 65% 2 MB |=========================================== | 66% 2 MB |============================================ | 67% 2 MB |============================================= | 69% 3 MB |============================================= | 70% 3 MB |============================================== | 71% 3 MB |=============================================== | 73% 3 MB |================================================ | 74% 3 MB |================================================= | 75% 3 MB |================================================== | 77% 3 MB |=================================================== | 78% 3 MB |=================================================== | 79% 3 MB |==================================================== | 81% 3 MB |===================================================== | 82% 3 MB |====================================================== | 83% 3 MB |======================================================= | 85% 3 MB |======================================================== | 86% 3 MB |========================================================= | 87% 3 MB |========================================================== | 89% 3 MB |========================================================== | 90% 3 MB |=========================================================== | 91% 3 MB |============================================================ | 93% 4 MB |============================================================= | 94% 4 MB |============================================================== | 95% 4 MB |=============================================================== | 97% 4 MB |================================================================| 98% 4 MB |================================================================| 99% 4 MB |=================================================================| 100% 4 MB Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_33_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_34_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_35_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_36_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) |== | 3% |=== | 4% |==== | 5% |===== | 7% |====== | 8% |======= | 9% |======= | 10% |======== | 12% |========= | 13% |========== | 14% |=========== | 15% |============ | 17% |============= | 18% |============== | 19% |=============== | 20% |=============== | 21% |=============== | 23% 1 MB |=============== | 24% 1 MB |================ | 25% 1 MB |================= | 26% 1 MB |================== | 28% 1 MB |=================== | 29% 1 MB |=================== | 30% 1 MB |==================== | 31% 1 MB |===================== | 32% 1 MB |====================== | 34% 1 MB |====================== | 35% 1 MB |======================= | 36% 1 MB |======================== | 37% 1 MB |========================= | 39% 1 MB |========================== | 40% 1 MB |=========================== | 41% 1 MB |=========================== | 42% 1 MB |============================ | 43% 1 MB |============================= | 45% 1 MB |============================== | 46% 2 MB |============================== | 47% 2 MB |=============================== | 48% 2 MB |================================ | 50% 2 MB |================================= | 51% 2 MB |================================== | 52% 2 MB |================================== | 53% 2 MB |=================================== | 55% 2 MB |==================================== | 56% 2 MB |===================================== | 57% 2 MB |====================================== | 58% 2 MB |======================================= | 60% 2 MB |======================================= | 61% 2 MB |======================================== | 62% 2 MB |========================================= | 63% 2 MB |========================================== | 65% 2 MB |=========================================== | 66% 2 MB |============================================ | 67% 2 MB |============================================= | 69% 3 MB |============================================= | 70% 3 MB |============================================== | 71% 3 MB |=============================================== | 73% 3 MB |================================================ | 74% 3 MB |================================================= | 75% 3 MB |================================================== | 77% 3 MB |=================================================== | 78% 3 MB |=================================================== | 79% 3 MB |==================================================== | 81% 3 MB |===================================================== | 82% 3 MB |====================================================== | 83% 3 MB |======================================================= | 85% 3 MB |======================================================== | 86% 3 MB |========================================================= | 87% 3 MB |========================================================== | 89% 3 MB |========================================================== | 90% 3 MB |=========================================================== | 91% 3 MB |============================================================ | 93% 4 MB |============================================================= | 94% 4 MB |============================================================== | 95% 4 MB |=============================================================== | 97% 4 MB |================================================================| 98% 4 MB |================================================================| 99% 4 MB |=================================================================| 100% 4 MB Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_37_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_38_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_39_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_4_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_40_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_41_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_42_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_43_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_44_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_45_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_46_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_47_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_48_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_49_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_5_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_50_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_6_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_7_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_8_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) exp_33_paravance_65_manual/scorep-RUN_513_STEP_9_of_50/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Warning: 10260 parsing failures. row # A tibble: 5 x 5 col row col expected actual expected actual 1 7696 bytes_sent an integer 6106101516 file 2 7696 bytes_received an integer 6106101516 row 3 7697 bytes_sent an integer 6106101516 col 4 7697 bytes_received an integer 6106101516 expected 5 7698 bytes_sent an integer 6106101516 actual # ... with 1 more variables: file ... ................. ... ............................................ ........ ............................................ ...... ............................................ .... ............................................ ... ............................................ ... ............................................ ........ ............................................ ...... ....................................... See problems(...) for more details. Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) There were 50 or more warnings (use warnings() to see the first 50) #+end_example #+begin_src R :results output :session :exports both exp33.ALL #+end_src #+RESULTS: #+begin_example # A tibble: 256,000 x 18 Phase Code ID Rank visits time min_time max_time bytes_put 1 1 Computation 14 1 1 3.405599 11.07904 11.07904 0 2 1 Computation 14 2 1 3.347164 11.07898 11.07898 0 3 1 Computation 14 3 1 3.393387 11.07907 11.07907 0 4 1 Computation 14 4 1 3.409869 11.07887 11.07887 0 5 1 Computation 14 5 1 3.303782 11.07908 11.07908 0 6 1 Computation 14 6 1 3.601905 11.07898 11.07898 0 7 1 Computation 14 7 1 3.286918 11.07910 11.07910 0 8 1 Computation 14 8 1 3.443075 11.07900 11.07900 0 9 1 Computation 14 9 1 3.302105 11.07908 11.07908 0 10 1 Computation 14 10 1 3.667202 11.07895 11.07895 0 # ... with 255,990 more rows, and 9 more variables: bytes_get , # ALLOCATION_SIZE , DEALLOCATION_SIZE , bytes_leaked , # maximum_heap_memory_allocated , bytes_sent , # bytes_received , NP , Seq #+end_example **** ALL Rough initial LB Analysis #+begin_src R :results output graphics :file img/exp33_ALL.png :exports both :width 1800 :height 400 :session exp33.ALL %>% mutate(NP = as.integer(NP), Seq = as.integer(Seq)) %>% filter(NP == 513) %>% group_by(NP, Seq, Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_line(aes(group=Rank), alpha=.1) + geom_point() + ylim(0,NA) + theme_bw(base_size=18) + facet_wrap(~Seq, nrow=2) #+end_src #+RESULTS: [[file:img/exp33_ALL.png]] **** Save data for Arnaud's analysis :ATTACH: :PROPERTIES: :Attachments: exp33_compute_ALL.csv.gz :ID: 5f287c9b-27f1-4fe6-a368-220504cba4ad :END: #+begin_src R :results output :session :exports both write_csv(exp33.ALL, "exp33_compute_ALL.csv.gz"); #+end_src #+RESULTS: **** TODO X. Verify the load imbalance (and eventually summarize it if necessary) ***** Rough initial LB Analysis #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 600 :height 400 :session df.TMP %>% group_by(Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_line(aes(group=Rank), alpha=.1) + geom_point() + ylim(0,NA) + theme_bw(base_size=18) #+end_src #+RESULTS: [[file:/tmp/babel-22415aoO/figure22415k9p.png]] ***** Calculate the average #+name: exp33_lb_average #+header: :var dep0=exp33_enrich #+begin_src R :results output :session :exports both exp.ENRICH #+end_src #+RESULTS: exp33_lb_average #+begin_example # A tibble: 64 x 2 Rank time 1 1 42.92296 2 2 64.98924 3 3 66.13120 4 4 62.14705 5 5 64.66830 6 6 59.88956 7 7 61.76306 8 8 67.09068 9 9 57.22863 10 10 64.70995 # ... with 54 more rows #+end_example **** TODO 8. Target with Inverting Hilbert Load Curve ***** The function #+name: exp33_hilbert_invertion_function #+begin_src R :results output :session :exports both df_cum_inverse = function(df,yval) { df = rbind(data.frame(x=0,y=0),df[names(df) %in% c("x","y")]) N = nrow(df); xval = rep(NA,length(yval)) # print(N); for(i in 1:length(yval)) { # print(yval[i]) if(yval[i]<=0) {xval[i] = 0;} else if(yval[i]>=1) {xval[i] = 1;} else { idx = df %>% mutate(Order = 1:nrow(.)) %>% filter(y >= yval[i]) %>% slice(1) %>% pull(Order) idx = idx-1; # print(idx) # print(df[idx:(idx+1),]) eps_x = (df[idx+1,]$x - df[idx,]$x); eps_y = (df[idx+1,]$y - df[idx,]$y); xval[i] = ((yval[i]-df[idx,]$y)*eps_x/eps_y + df[idx,]$x); } } return(xval); } #+end_src #+RESULTS: exp33_hilbert_invertion_function ***** Calculate the Target #+name: exp33_target #+header: :var dep0=exp33_cumsum #+header: :var dep1=exp33_hilbert_invertion_function #+begin_src R :results output :session :exports both df.CUMSUM %>% rename(y = TIMESUM, x = NELEMSUM) -> df.CUMSUM.2; nb_proc = nrow(df.CUMSUM.2) df_cum_inverse(df.CUMSUM.2, y = (1:nb_proc)/nb_proc) -> l.TARGET; df.CUMSUM.2 %>% mutate(Target.x = l.TARGET, Target.y = (1:nb_proc)/nb_proc) -> df.TARGET; df.TARGET; #+end_src #+name: exp33_tidying #+header: :var dep0=exp33_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, x, y) -> df.1; df.TARGET %>% select(Rank, Target.x, Target.y) -> df.2; df.1 %>% mutate(Category = "Observed") %>% bind_rows(df.2 %>% mutate(Category = "Target") %>% rename(x = Target.x, y = Target.y)) -> df.TARGET.tidy; df.TARGET %>% select(x, y, Target.x, Target.y) %>% filter(x == Target.x) #+end_src #+RESULTS: : # A tibble: 1 x 4 : x y Target.x Target.y : : 1 1 1 1 1 ***** Plot with the arrows #+header: :var dep0=exp33_tidying #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 800 :height 400 :session m1=0 m2=m1+1 NP = df.TARGET.tidy %>% pull(Rank) %>% unique %>% length df.TARGET.tidy %>% ggplot(aes(x=x, y=y, group+category)) + theme_bw(base_size=20) + geom_curve (data=df.TARGET %>% filter(x!=1), aes(x=x, y=y, xend=Target.x, yend=Target.y), arrow=arrow(length = unit(0.03, "npc"))) + geom_point(size=1, aes(shape=Category, color=as.factor(Rank%%8))) + scale_color_brewer(palette = "Set1") + # geom_line(data=df.TARGET.tidy %>% filter(Category == "Observed")) + coord_cartesian(xlim=c(m1, m2), ylim=c(m1, m2)) #+ # geom_hline(yintercept=(0:NP)/NP) #+end_src #+RESULTS: [[file:/tmp/babel-16900S5M/figure16900T5J.png]] **** TODO 9. Output the new =rank-elements.dat= file #+header: :var dep0=exp33_target #+begin_src R :results output :session :exports both df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% write_delim("rank-elements.dat", delim=" ", col_names=FALSE) #+end_src #+RESULTS: **** TODO 9.1 Check which rank has the larger correction #+header: :var dep0=exp33_target #+begin_src R :results output :session :exports both t = 61 df.TARGET %>% select(Rank, Target.x) %>% mutate(Target.x.aux = c(0, Target.x[-n()])) %>% mutate(Target = (Target.x - Target.x.aux) * NP) %>% select(Rank, Target) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 2 : Rank Target : : 1 61 0.9131876 : 2 62 1.1703797 : 3 63 1.0715776 : 4 64 1.0107468 #+begin_src R :results output :session :exports both df.INTEGRATED %>% mutate(F = time / NELEM) %>% mutate(diff = mean(time) - time) %>% filter(Rank >= t) #+end_src #+RESULTS: : # A tibble: 4 x 5 : Rank time NELEM F diff : : 1 61 72.95337 141232 0.0005165499 -10.41747 : 2 62 81.25564 221395 0.0003670166 -18.71973 : 3 63 29.42375 162676 0.0001808734 33.11215 : 4 64 66.49157 157024 0.0004234484 -3.95566 #+begin_src R :results output :session :exports both exp.ENRICH %>% pull(Rank) %>% min exp.ENRICH %>% summary exp.ENRICH %>% pull(time) %>% sd exp.ENRICH %>% arrange(-time) #+end_src #+RESULTS: #+begin_example [1] 1 Rank time Min. : 1.00 Min. : 32.55 1st Qu.:16.75 1st Qu.: 59.81 Median :32.50 Median : 61.14 Mean :32.50 Mean : 61.80 3rd Qu.:48.25 3rd Qu.: 63.00 Max. :64.00 Max. :103.14 [1] 12.66165 # A tibble: 64 x 2 Rank time 1 31 103.14403 2 59 97.99925 3 46 96.40160 4 44 91.13452 5 3 81.42977 6 20 77.99168 7 1 67.54315 8 36 66.69546 9 56 65.95448 10 39 65.71467 # ... with 54 more rows #+end_example ** Quick analysis of whether NELEM, NPOIN, NBOUN, NPOI32 could be used as a predictor :EXP26:EXP25:EXP24: Let's load the data Lucas sent me: #+begin_src R :results output :session *R* :exports both suppressMessages(library(tidyverse)); df_params = read_csv("exp24-exp25-exp26.csv.gz") df_params #+end_src #+RESULTS: #+begin_example Parsed with column specification: cols( Phase = col_integer(), ID = col_integer(), Rank = col_integer(), NELEM = col_integer(), NPOIN = col_integer(), NBOUN = col_integer(), NPOI32 = col_integer(), Exp = col_character() ) # A tibble: 704 x 8 Phase ID Rank NELEM NPOIN NBOUN NPOI32 Exp 1 1 14 1 190452 37438 4546 2406 exp24 2 1 14 2 120468 45173 11308 3570 exp24 3 1 14 3 134197 43452 8856 4330 exp24 4 1 14 4 133436 40913 7662 3896 exp24 5 1 14 5 130623 42986 9030 3931 exp24 6 1 14 6 145601 43436 7358 4927 exp24 7 1 14 7 130407 44445 9289 5185 exp24 8 1 14 8 127508 43276 9881 3418 exp24 9 1 14 9 134033 41766 7851 4177 exp24 10 1 14 10 141351 40484 6690 3872 exp24 # ... with 694 more rows #+end_example Let's load all the corresponding traces. #+header: :var dep0=exp24_cumsum #+header: :var dep1=exp25_cumsum #+header: :var dep2=exp26_cumsum #+begin_src R :results output :session *R* :exports both df.exp24.CUMSUM %>% mutate(Type = "exp24") %>% bind_rows(df.exp25.CUMSUM %>% mutate(Type = "exp25")) %>% bind_rows(df.exp26.CUMSUM %>% mutate(Type = "exp26")) -> df_time df_time #+end_src #+RESULTS: #+begin_example # A tibble: 192 x 6 Rank time NELEM TIMESUM NELEMSUM Type 1 1 88.8 190452 0.0220 0.0207 exp24 2 2 33.1 120468 0.0302 0.0338 exp24 3 3 63.3 134197 0.0459 0.0483 exp24 4 4 35.4 133436 0.0546 0.0628 exp24 5 5 37.2 130623 0.0639 0.0770 exp24 6 6 42.2 145601 0.0743 0.0928 exp24 7 7 61.3 130407 0.0895 0.107 exp24 8 8 62.4 127508 0.105 0.121 exp24 9 9 72.4 134033 0.123 0.135 exp24 10 10 59.6 141351 0.138 0.151 exp24 # ... with 182 more rows #+end_example #+begin_src R :results output :session *R* :exports both df_time %>% select(-NELEM) %>% left_join(df_params %>% filter(Phase == 1), by=c("Type" = "Exp", "Rank" = "Rank")) -> df_merged_lm df_merged_lm %>% group_by(Type) %>% summarize(time.mean = mean(time), time.max = max(time), time.min = min(time)) #+end_src #+RESULTS: : # A tibble: 3 x 4 : Type time.mean time.max time.min : : 1 exp24 63.1 105 30.5 : 2 exp25 62.0 102 35.7 : 3 exp26 61.8 103 32.5 Then using information from https://stackoverflow.com/questions/1169539/linear-regression-and-group-by-in-r #+begin_src R :results output :session *R* :exports both library(broom) df_merged_lm %>% group_by(Type) %>% do(model = lm(time ~ (NELEM + NPOIN + NBOUN + NPOI32), data = .)) -> reg #+end_src #+RESULTS: Let's have a look at the R^2: #+begin_src R :results output :session *R* :exports both reg %>% glance(model) #+end_src #+RESULTS: : # A tibble: 3 x 12 : # Groups: Type [3] : Type r.squared adj.r.squared sigma statistic p.value df logLik AIC : : 1 exp24 0.368 0.325 12.9 8.58 0.0000158 5 -252 515 : 2 exp25 0.315 0.268 12.4 6.77 0.000148 5 -250 511 : 3 exp26 0.196 0.142 11.7 3.60 0.0108 5 -246 504 : # ... with 3 more variables: BIC , deviance , df.residual Yuck! Do we have significant parameters? #+begin_src R :results output :session *R* :exports both reg %>% tidy(model) %>% mutate(signif=ifelse(p.value<.001, "***", ifelse(p.value<.01, "**", ifelse(p.value<.05, "*", ifelse(p.value<.1, ".", ""))))) #+end_src #+RESULTS: #+begin_example # A tibble: 15 x 7 # Groups: Type [3] Type term estimate std.error statistic p.value signif 1 exp24 (Intercept) 47.9 11.4 4.20 0.0000919 *** 2 exp24 NELEM 0.000588 0.000150 3.93 0.000223 *** 3 exp24 NPOIN - 0.00212 0.000849 -2.50 0.0153 * 4 exp24 NBOUN 0.00218 0.00169 1.30 0.200 "" 5 exp24 NPOI32 0.00110 0.00255 0.431 0.668 "" 6 exp25 (Intercept) 34.9 9.54 3.66 0.000540 *** 7 exp25 NELEM - 0.000213 0.000189 -1.13 0.265 "" 8 exp25 NPOIN 0.00183 0.000949 1.92 0.0592 . 9 exp25 NBOUN - 0.00214 0.00181 -1.18 0.243 "" 10 exp25 NPOI32 - 0.000908 0.00253 -0.358 0.721 "" 11 exp26 (Intercept) 32.8 8.53 3.84 0.000303 *** 12 exp26 NELEM 0.000235 0.000135 1.74 0.0863 . 13 exp26 NPOIN - 0.000350 0.000755 -0.464 0.645 "" 14 exp26 NBOUN 0.00104 0.00150 0.693 0.491 "" 15 exp26 NPOI32 0.000378 0.00216 0.175 0.862 "" #+end_example Not really. :( Let's have a look at the residuals to check whether they are structured or not. #+begin_src R :results output graphics :file (org-babel-temp-file "figure" ".png") :exports both :width 1600 :height 400 :session *R* reg %>% augment(model) %>% ggplot(aes(x=.fitted,y=.resid)) + facet_wrap(~Type) + geom_smooth(color="red") + geom_point() #+end_src #+RESULTS: [[file:/tmp/babel-18292gz_/figure182927Vd.png]] ** 10-node (160 cores) rennes parasilo, many rounds (after MN4) :EXP34: *** Execution script #+begin_src shell :results output # Configuration part export STEPS=10 export TIMESTEPS=5 export EXPEKEY=exp_34_parasilo_10_manual export EXPEDIR=${HOME}/${EXPEKEY} export MACHINEFILE=$HOME/parasilo export CASEDIR=$HOME/WORK-RICARD/resp_sfc/ export CASENAME=fensap export ALYA=$HOME/alya-bsc-sfc/Executables/unix/Alya.x rm -rf $EXPEDIR mkdir -p $EXPEDIR export GITREPO=$HOME/Alya-Perf/ # Control part cat $MACHINEFILE | uniq > /tmp/mf-uniq $GITREPO/scripts/control_experiment.sh /tmp/mf-uniq $EXPEDIR # Use spack source ~/spack-ALYA/share/spack/setup-env.sh # Use the correct mpicc/mpif90/mpirun export PATH=$(spack location -i openmpi)/bin:$PATH # Use the cube_dump for the refinement workflow export PATH=$(spack location -i cube)/bin:$PATH pushd $CASEDIR sed -i "s/NUMBER_OF_STEPS=.*$/NUMBER_OF_STEPS=${TIMESTEPS}/" fensap.dat for NP in 160; do # Generate initial rank-elements.dat to case for i in $(seq 1 $(echo ${NP} - 1 | bc)); do echo $i 1 done > $CASEDIR/rank-elements.dat for RUN in $(seq 1 ${STEPS}); do RUNKEY="RUN_${NP}_STEP_${RUN}_of_${STEPS}" SCOREPDIR="scorep-${RUNKEY}" rm -rf $SCOREPDIR echo $RUNKEY # Run the program $(which mpirun) \ --mca btl_base_warn_component_unused 0 \ --bind-to core:overload-allowed \ --report-bindings \ -x SCOREP_TOTAL_MEMORY=3900MB \ -x SCOREP_MPI_ENABLE_GROUPS=ALL \ -x SCOREP_ENABLE_TRACING=FALSE \ -x SCOREP_ENABLE_PROFILING=TRUE \ -x SCOREP_EXPERIMENT_DIRECTORY=$SCOREPDIR \ -x LD_LIBRARY_PATH=$(spack location -i openmpi)/lib/ \ -np ${NP} \ -machinefile $MACHINEFILE \ $ALYA $CASENAME # Save the data into the SCOREPDIR cp $MACHINEFILE $SCOREPDIR cp *.log ${CASENAME}.dat $SCOREPDIR mv domain-*.csv iterations-*.csv $SCOREPDIR cp rank-elements.dat $SCOREPDIR # Run the refinement workflow pushd $SCOREPDIR cp $GITREPO/Refinement_Workflow.org . emacs -batch -l ~/.emacs.d/init.el \ --eval "(setq vc-follow-symlinks nil)" \ --eval "(setq ess-ask-for-ess-directory nil)" \ --eval "(setq org-export-babel-evaluate t)" \ Refinement_Workflow.org \ --funcall org-babel-execute-buffer # Compress data (after workflow) gzip *.csv # Adjust rank-elements_next.dat NRANKS=rank-elements_next.dat cp $NRANKS $CASEDIR/rank-elements.dat popd # Move all data to EXPEDIR for archiving purposes mv $SCOREPDIR $EXPEDIR done done > execution.log mv execution.log $EXPEDIR popd #+end_src *** Data Transformation **** Read one DIR function #+name: exp34_read_one_dir #+begin_src R :results output :session :exports both suppressMessages(library(tidyverse)); read_one_DIR <- function(DIR) { # Get basic info about the experience tibble(Dir=DIR) %>% separate(Dir, into=c("X", "Case"), sep="/") %>% select(-X) %>% separate(Case, into=c("X1", "NP", "X3", "Seq", "X5", "X6"), sep="_") %>% select(-X1, -X3, -X5, -X6) -> basic.info; print(basic.info); # 2. Transform profile.cubex PROFILE.CUBEX = paste0(DIR, "/profile.cubex"); PROFILE.CSV = paste0(DIR, "/profile.csv"); REGIONSCODES.CSV = paste0(DIR, "/regions-codes.csv.gz"); CUBEINFO.TXT = paste0(DIR, "/cube_info.txt"); CUBEINFO.CSV = paste0(DIR, "/cube_info.csv.gz"); CUBE.DUMP = "/home/schnorr/install/stow/bin//cube_dump" system2(CUBE.DUMP, args=paste("-c all -m all -s csv2", PROFILE.CUBEX, ">", PROFILE.CSV)); system2(CUBE.DUMP, args=paste("-w", PROFILE.CUBEX, "| tail -n+34 | head -n283 | tr '(' ';' | sed -e \"s/[[:space:]]*//g\" -e \"s/,.*$//\" -e \"s/id=//\" -e \"s/:[^;]*;/;/\" > ", REGIONSCODES.CSV)); system2(CUBE.DUMP, args=paste("-w", PROFILE.CUBEX, ">", CUBEINFO.TXT)); system2("./scripts/cube_calltree.pl", args=DIR); # 1. Read Iteration Timings if(FALSE){ do.call("bind_rows", lapply(list.files(DIR, pattern="iterations-.*csv.gz$", full.names=TRUE), function(file) { read_table(file, col_names=FALSE) %>% rename(Time.User = X1, Time.System = X2, Time.Run = X3, Rank = X4, Iteration = X5) %>% filter(Iteration != 0) %>% group_by(Rank, Iteration) %>% summarize(Start = min(Time.User), End = max(Time.User)) %>% group_by(Rank) %>% mutate(End = End - min(Start), Start = Start - min(Start)) })) -> exp.iter; } # 4. Enrich the call tree df.PROF <- read_csv(PROFILE.CSV); exp.REGION <- read_delim(REGIONSCODES.CSV, col_names=FALSE, delim=";") %>% rename(ID = X2, Name = X1); exp.ENRICH <- read_delim(CUBEINFO.CSV, delim=",", col_names=FALSE) %>% rename(Phase = X1, Code = X2, ID = X3) %>% left_join(df.PROF, by=c("ID" = "Cnode ID")) %>% rename(Rank = `Thread ID`) %>% filter(Code == "Computation") %>% mutate(Phase = as.integer(Phase)) %>% filter(Rank != 0) %>% mutate(NP = basic.info$NP, Seq = basic.info$Seq) return(exp.ENRICH); # 5. The number of elements on T1 NOELEMENTS.CSV = "no_elements.csv"; system2("./scripts/no_elements_domain.sh", args=paste(DIR, NOELEMENTS.CSV)); exp.ELEMENTS <- read_csv(paste0(DIR, "/", NOELEMENTS.CSV)) %>% filter(Rank != 0) # Calculate the average exp.ENRICH %>% filter(Phase <= 9) %>% group_by(Rank) %>% summarize(time = mean(time)) -> exp.ENRICH.AVERAGE; # 6. Integrate number of elements with compute cost exp.ENRICH.AVERAGE %>% left_join(exp.ELEMENTS) %>% select(Rank, time, NELEM) -> df.INTEGRATED; # 7. Do the cumsum df.INTEGRATED %>% arrange(Rank) %>% mutate(TIMESUM = cumsum(time)/sum(time)) %>% mutate(NELEMSUM = cumsum(NELEM)/sum(NELEM)) -> df.CUMSUM; } #+end_src #+RESULTS: exp34_read_one_dir **** Read ALL DIRs #+begin_src R :results output :session :exports both do.call("bind_rows", lapply(list.files("exp_34_parasilo_10_manual", pattern="scorep", full.names=TRUE), function(case) { read_one_DIR(case); } )) -> exp34.ALL; #+end_src #+RESULTS: #+begin_example # A tibble: 1 x 2 NP Seq ,* 1 160 1 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_1_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 10 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_10_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 2 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_2_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 3 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_3_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 4 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_4_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 5 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_5_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 6 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_6_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 7 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_7_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 8 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_8_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) # A tibble: 1 x 2 NP Seq ,* 1 160 9 exp_34_parasilo_10_manual/scorep-RUN_160_STEP_9_of_10/cube_info.csv Parsed with column specification: cols( `Cnode ID` = col_integer(), `Thread ID` = col_integer(), visits = col_integer(), time = col_double(), min_time = col_double(), max_time = col_double(), bytes_put = col_integer(), bytes_get = col_integer(), ALLOCATION_SIZE = col_integer(), DEALLOCATION_SIZE = col_integer(), bytes_leaked = col_integer(), maximum_heap_memory_allocated = col_integer(), bytes_sent = col_integer(), bytes_received = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_integer() ) Parsed with column specification: cols( X1 = col_character(), X2 = col_character(), X3 = col_integer() ) #+end_example **** ALL Rough initial LB Analysis #+begin_src R :results output graphics :file img/exp34_ALL.png :exports both :width 1000 :height 400 :session exp34.ALL %>% mutate(NP = as.integer(NP), Seq = as.integer(Seq)) %>% group_by(NP, Seq, Phase, Rank, Code) %>% summarize(Sum = sum(time)) %>% ungroup() %>% filter(!is.na(Phase)) %>% ggplot(aes(x=as.integer(Phase), y=Sum, color=Rank)) + geom_line(aes(group=Rank), alpha=.1) + geom_point() + ylim(0,NA) + theme_bw(base_size=18) + facet_wrap(~Seq, nrow=2) #+end_src #+RESULTS: [[file:img/exp34_ALL.png]]