#!/usr/bin/env python # This library contains functions for creating part of the # afni_proc.py QC HTML. Specifically, these functions build the # image, data and text of the single-subject HTML review. # # auth = 'PA Taylor' # ver : 1.32 || date: Oct 15, 2018 # + some level of written-ness # # ver : 1.33 || date: Oct 16, 2018 # + new uvars # + new checks # + new QC dir and subdir defs # # ver : 1.34 || date: Oct 17, 2018 # + new uvars # + new text, new output strings # + WARN type introduced # #ver = '1.4' ; date = 'Oct 23, 2018' # + do stats differently: separate olay and thr # + also start a json for the pbar info # #ver = '1.5' ; date = 'Nov 1, 2018' # + update regression warning # #ver = '1.51' ; date = 'Nov 19, 2018' # + update 1dplot string labels (add in "enorm" and "outliers") # #ver = '1.6' ; date = 'Dec 2, 2018' # + check_dep() now has more conditions, because of some specific # vars+values that may or may not be there # - first one: for stats_dsets, check about "NO_STATS" value # #ver = '1.7' ; date = 'Dec 23, 2018' # + [PT] put the censoring vertical color bars into the reg* 1D plots, # for both basic and pythonic styles # + [PT] rename "run_mode" -> "run_style" (more specific/correct) # #ver = '1.8' ; date = 'Jan 2, 2019' # + [PT] add media_info # + [PT] change order of enorm and outlier # + [PT] add in EPI in orig space viewing # + [PT] copy gen_ss JSON file into QC_*/media_info/ # #ver = '2.1' ; date = 'Feb 26, 2019' # + [PT] new plot in regr: grayplot # #ver = '2.2' ; date = 'May 14, 2019' # + [PT] new images: radcor # #ver = '2.21' ; date = 'May 16, 2019' # + [PT] correctifying radcor behavior # #ver = '2.21' ; date = 'May 17, 2019' # + [PT] simplifying radcor behavior # #ver = '2.4' ; date = 'May 20, 2019' # + [PT] more details of aea_checkflip # + [PT] radcor to own QC block # #ver = '2.5' ; date = 'May 23, 2019' # + [PT] switched to using afni_base functions for executing on # commandline # #ver = '2.7' ; date = 'June 26, 2019' # + [PT] elif for vstat: no anat or templ, use volreg as ulay; West # Coast usage for S Torrisi. # #ver = '2.8' ; date = 'June 28, 2019' # + [PT] better grayplotting, more informative stuff happening # + [PT] PBAR size is now in terms of char width-- preserve verticality. Duh. # #ver = '2.9' ; date = 'July 3, 2019' # + [PT] vorig block now starting to be used # + [PT] add in more stats to be viewed # + [PT] add in QC block ID to QC block titles # #ver = '2.91' ; date = 'July 3, 2019' # + [PT] bannerize now has 'padsymb=' kwarg # #ver = '3.0' ; date = 'July 18, 2019' # + [PT] include obliquity in vorig QC block # + [PT] simplify radcor text; decrease repetition # + [PT] -> merge in changed opts for radcor # #ver = '3.1' ; date = 'Sep 6, 2019' # [PT] put a montgap (1 line, black) into QC montages: sep imgs a bit # + put in censoring to the 1dplot.py command when showing VR6 - # also known as the 'Molfese approach' # #ver = '3.11' ; date = 'Sep 9, 2019' # [PT] spacing fix in VR6 with censoring # #ver = '3.12' ; date = 'Dec 26, 2019' # [PT] for regr QC block, indiv stim plotting: don't need 'xmat_uncensored' # as a dependency, so remove it from the list # #ver = '3.2' ; date = 'Jan 9, 2020' # [PT] new warning block: censor fraction # + seed point plotting introduced (vstat section for resting state) # #ver = '3.21' ; date = 'Jan 9, 2020' # [PT] raise the thresholds in the vstat_seedcorr regr_corr plots # + so much above threshold otherwise, including noise # #ver = '3.22' ; date = 'Jan , 2020' # [PT] fix type conversion error of scalars -> lists # #ver = '3.3' ; date = 'Feb 15, 2020' # [PT] new funcs for 'widely used' params # + for censor and sundry info. # #ver = '3.31' ; date = 'Feb 17, 2020' # [PT] further cleaned up (simplified?) a lot of the censoring info # #ver = '3.32' ; date = 'Feb 21, 2020' # [PT] fix minor bug in case of: 'basic' html with no outlier-based censoring # #ver = '3.33' ; date = 'Feb 26, 2020' # [PT] fix minor bug in case of: 'pythonic' html with no censoring at all. Sigh. # #ver = '3.4' ; date = 'March 11, 2020' # [PT] change way template/final_anat dsets are proc'ed/used. # + new top level section to get template/anat_final properties # + va2t: now underlay anat, and use template for edges # + vstat: now underlay template (if there), instead of anat_final # + regr: use template as ulay (if there), instead of anat_final # #ver = '3.41' ; date = 'March 12, 2020' # [PT] no vstat if 'surf' block was used in AP (-> stats dset is # *.niml.dset) # #ver = '3.5' ; date = 'March 27, 2020' # [PT] remove dependency on lib_apqc_html_helps.py # #ver = '3.6' ; date = 'May 26, 2020' # [PT] ve2a and LR-flipcheck now show EPI under anat edges # #ver = '3.61' ; date = 'May 28, 2020' # [PT] in vstat maps, report DF value(s) # #ver = '3.62' ; date = 'May 31, 2020' # [PT] EPI ulay ranges in ve2a and LR-flipcheck now: NZ 2-98% # #ver = '3.63' ; date = 'May 31, 2020' # [PT] vstat seedbased corr seed thr from 0.3 -> 0.2 # #ver = '3.64' ; date = 'June 14, 2020' # [PT] return vstat seedbased corr seed thr to 0.3 (from 0.2) # + if ~normal smoothing is done, this is needed # + if no smoothing is done # #ver = '3.65' ; date = 'July 30, 2020' # [PT] # + search for template first with full given path, then just by # basename. this makes it easier to have QC work if directory # structure changes (e.g., for demos, that are downloaded+run on # different computers) # + also include a wildcard to help clean intermed file, in case # auto GZIP is on # #ver = '3.7' ; date = 'Feb 22, 2021' # [PT] add in TSNR images # #ver = '3.71' ; date = 'Feb 22, 2021' # [PT] updates with TSNR images/fnames # + following on from AP updates from RCR---thanks! # #ver = '3.72' ; date = 'Feb 24, 2021' # [PT] TSNR image no longer *requires* mask # + add a type-check to dep_check. # #ver = '3.73' ; date = 'Mar 5, 2021' # [PT] cp review basic text to QC_*/ dir # #ver = '3.74' ; date = 'Apr 6, 2021' # [PT] update TSNR-vreg checks # + give sep names for TSNR images: tsnr_vreg and tsnr_fin # #ver = '3.75' ; date = 'Apr 6, 2021' # [PT] now use adjunct*tsnr*general prog (just added, only need 1 prog) # #ver = '3.76' ; date = 'Sep 21, 2021' # [PT] use '-no_cor' to not make coronal plane images # + save nearly 33% of space in QC_${subj} dir # #ver = '3.77' ; date = 'Sep 21, 2021' # [PT] adjunct*tsnr: '-no_cor' to not make coronal plane images # + keep applying new opt # #ver = '3.78' ; date = 'Sep 27, 2021' # [PT] Due to recent changes (from ~Aug 23) in label_size defaults # in imseq.c, adjust the default labelsize from 3 -> 4. # + this should restore labels to their longrunning size (since Aug # 23 they have been one size smaller by default); but the new font # will be bolder than previously, due to those imseq.c changes. # #ver = '3.80' ; date = 'Jan 20, 2022' # [PT] move the parts of code for task-based FMRI vstat selection to a # new library: lib_apqc_stats_dset.py # + as part of this, adding in fuller stats representation by default. # #ver = '3.93' ; date = 'Jan 26, 2022' # [PT] epi-anat overlap in vorig QC block # #ver = '4.0' ; date = 'June 5, 2022' # [PT] ve2a: better %ile range for ulay: should have have better contrast # + ve2a: also introduce scaling/values for local-unifized EPI as ulay # #ver = '4.01' ; date = 'June 6, 2022' # [PT] ve2a: new scaling for ulay, extra control of grayscale with # ulay_min_fac # #ver = '4.02' ; date = 'July 27, 2022' # [PT] mecho: cp -> rsync, because of annoying Mac difference in cp # #ver = '4.03' ; date = 'Aug 18, 2022' # [PT] add warns: 3dDeconvolve *.err text file # #ver = '4.04' ; date = 'Aug 18, 2022' # [PT] add mask_dset images: overlays final dset, whether in # va2t, ve2a or vorig QC block # #ver = '4.05' ; date = 'Aug 18, 2022' # [PT] put already-calc'ed Dice info below ve2a and va2t olay imgs # ---> but just as quickly have removed it; might distract from the # important sulcal/gyral overlap # #ver = '4.06' ; date = 'Aug 31, 2022' # [PT] make a JSON version of ss_rev_basic TXT file in QC*/extra_info # -> will use this for 'saving' mode of APQC HTML interaction # #ver = '4.1' ; date = 'Nov 15, 2022' # [PT] many new parts for var_lines (AKA vlines) and tcat QC # #ver = '5.0' ; date = 'Mar 05, 2023' # [PT] move toward Python-only implementation, rather than generating # a script intermediately, to simplify flexibility, additions and # apqc2/NiiVue functionality # #ver = '5.1' ; date = 'Aug 22, 2023' # [PT] fix Py2-Py3 compatibility: flush print funcs separately # #ver = '5.2' ; date = 'Aug 22, 2023' # [PT] fix Py2-Py3 compatibility: open(...) function stuff, for utf-8 enc # + a bit more with print functions, too # #ver = '5.3' ; date = 'Aug 31, 2023' # [PT] fix initialization of some 1dplot/1dplot.py variables, for full # use cases (namely when censoring levels are *not* set # + thanks for pointing these out, C Rorden! # #ver = '6.0' ; date = 'Mar 19, 2024' # [PT] use new chauffeur functionality where run_* scripts have 1x1 mont # ver = '6.1' ; date = 'Feb 7, 2025' # [PT] expand functionality when part of analysis has not been done in AP # ######################################################################### import os, copy import sys import glob import json import codecs import subprocess import collections as coll from datetime import datetime from afnipy import afni_base as ab from afnipy import afni_util as au from afnipy import lib_apqc_html as lah from afnipy import lib_apqc_stats_dset as lasd from afnipy import lib_ss_review as lssr from afnipy import lib_apqc_io as laio from afnipy import lib_apqc_niivue as lanv # ---------------------------------------------------------------------- ohtml = 'index.html' # output file, HTML page scriptname = '@ss_review_html' qcbase = 'QC' # full odir has subj ID concatenated dir_info = 'extra_info' # for gen_ss- and AP-JSONs, and more page_title_json = '__page_title' # used in one of the warnings checks fname_vlines_img = 'QC_var_lines.jpg' fname_vlines_txt = 'QC_var_lines.txt' # logo files all_logo = [ 'apqc_logo_main.svg', 'apqc_logo_help.svg', ] # font info all_font = [ 'FiraCode-Bold.woff2', 'FiraCode-Regular.woff2', ] # ---------------------------------------------------------------------- coord_opp = { 'R' : 'L', 'L' : 'R', 'A' : 'P', 'P' : 'A', 'I' : 'S', 'S' : 'I', } def coord_to_gen_sys(x, order='RAI'): ''' Input ----- x : list of three coords, either float or str to be turned into float order : (opt) coord order (def = 'RAI') Output ------ out : list of 3 strings with generalized coord vals, e.g.: [-4R, 3A, 27S] ''' N = len(x) if len(x) != 3 : print("** ERROR: need x to be list of 3 coord values") sys.exit(4) out = [] for i in range(N): val = float(x[i]) if val > 0 : out.append(str(abs(val))+coord_opp[order[i]]) else: out.append(str(abs(val))+order[i]) return out # -------------------------------------------------------------------- def read_in_txt_to_dict(fname, tmp_name='__tmp_txt2json.json', DO_CLEAN=True) : '''Take a colon-separate file 'fname', convert it to a JSON file 'tmp_name', and then return the dictionary created thereby. Cleaning text file is optional (def: clean it) ''' if not(os.path.isfile(fname)): print("** ERROR: could not find text file: {}".format(fname)) return {} odict = {} cmd = '''abids_json_tool.py \ -overwrite \ -txt2json \ -prefix {outp} \ -input {inp} '''.format( inp = fname, outp = tmp_name ) com = ab.shell_com(cmd, capture=1) com.run() # get dictionary form of json with open(tmp_name, 'r') as fff: odict = json.load(fff) if DO_CLEAN : # rm tmp json cmd = '''\\rm {outp}'''.format( outp = tmp_name ) com = ab.shell_com(cmd, capture=1, save_hist=0) com.run() return odict # -------------------------------------------------------------------- def get_path_abin(): do_cap = True cmd = '''dirname `which apqc_make_tcsh.py`''' com = ab.shell_com(cmd, capture=do_cap) stat = com.run() abin_path = com.so[0] return abin_path # -------------------------------------------------------------------- def get_space_from_dset( dset ): cmd = '''@FindAfniDsetPath -full_path -append_file -space {}'''.format(dset) com = ab.shell_com(cmd, capture=1, save_hist=0) com.run() dset_fullpath = com.so[0] cmd = '''3dinfo -space {}'''.format(dset_fullpath) com = ab.shell_com(cmd, capture=1, save_hist=0) com.run() space = com.so[0] return space # -------------------------------------------------------------------- def is_volumetric( name ) : """Return True if the dataset is volumetric (i.e., BRIK/HEAD or NIFTI) and False if it is something else.""" # what could a volumetric extension in the uvar json be? all_vol_ext = ('.HEAD', '.nii', '.nii.gz') return name.endswith(all_vol_ext) # -------------------------------------------------------------------- def get_warn_level_3( val, cutoff_list=[] ): '''Go through simple conditional cases for assigning level of warning; cutoff_list must be 3 items here. List should be in order of descending severity. ''' if not(cutoff_list) : print("** ERROR: no cutoff list provided ") sys.exit(3) try: N = len(cutoff_list) if N != 3 : print("** ERROR: len of cutoff_list is {}, not 3".format(N)) sys.exit(3) except : print("** ERROR: cutoff_list entry not a list, but of type {}?".format(type(cutoff_list))) sys.exit(3) # parse text file for warning severity if val >= cutoff_list[0] : out = "severe" elif val >= cutoff_list[1] : out = "medium" elif val >= cutoff_list[2] : out = "mild" else : out = "none" return out # -------------------------------------------------------------------- def check_if_niml_dset(D, x): '''Does x in dictionary 'D' appear to be a *.niml.dset? (and hence on the surface)? Here, x is just a string, not a list. ''' if type(x) != str : sys.exit("** ERROR: input to check_if_niml_dset() needs to be a str, " "not: {}".format(type(x))) # general check: based on existence if not(x in D) : return 0 # check if string is long enough if len(D[x]) < 10 : return 0 if D[x][-10:] == ".niml.dset" : return 1 return 0 def check_dep(D, lcheck): '''Does dictionary 'D' contain each of the elements of 'lcheck'?''' if type(lcheck) != list : sys.exit("** ERROR: input to check_dep() needs to be a list, not:\n" " {}".format(type(lcheck))) HAS_ALL = 1 for x in lcheck: # general check: based on existence if not(D.__contains__(x)) : HAS_ALL = 0 break # ---- specific check(s), based on known values: # in this case, the $stats_dset var can contain essentially a # NULL value, NO_STATS, and that is reflected here elif x == "stats_dset" : if D[x] == "NO_STATS" : HAS_ALL = 0 break # also need to consider case that stats is on surface; at # the moment, won't have any QC imaging for this scenario elif D[x][-10:] == ".niml.dset" : HAS_ALL = 0 break elif x == "have_radcor_dirs" : if D[x] == "no" : HAS_ALL = 0 break return HAS_ALL # ---------------------------------------------------------------------- def bannerize( x, fullwid=76, indent=0, padpre=1, padpost=2, padsymb='='): '''Make the banner for each QC image block. Inputs ------ x : a string that is the message to put, and it gets wrapped with a comment symbol and '=' (or what user specifies). fullwid : can be any number of width; default is 76 because there are 2 other characters put in: ' ' before/after the string. indent : number of spaces to prepend to the line, if desired padpre : number of empty lines to affix beforehand padpost : number of empty lines to affix after text Returns ------- string : x padded with comment/lines/spaces. ''' out = indent*' ' + "# " x = x.strip() freespace = fullwid - len(out) - len(x) if freespace < 0: freespace = 0 lban = freespace // 2 rban = freespace - lban out+= lban * padsymb out+= ' '+x+' ' out+= rban * padsymb if padpre: out = padpre*'\n'+out if padpost: out+= padpost*'\n' return out # ---------------------------------------------------------------------- def padassign(x, L): '''Move an assignment operator '=' rightward to a new index L, if it exists to the left; if it exists to the right of that spot, or if it doesn't appear in that string, just return the original string. If multiple occurrences of '=' occur in a string, just return original. ''' # check if string contains it if not('=' in x): return x # check if multiple occurrences if x.count('=') > 1: return x # check if we are already there/rightward K = x.index('=') if K >= L: return x out = x.replace('=', (L-K)*' '+'=') return out # ---------------------------------------------------------------------- def commandize( x, fullwid=76, indent=0, padpre=0, padpost=0, ALIGNASSIGN=False, ALLEOL=True, cmdindent=4, REP_TIL=True): '''Make the readable/spaced/EOL-char'ed version for each QC image cmd. Inputs ------ x : a string that is the cmd; should arranged into multiple lines (for long commands). Don't need/shouldn't have EOL chars. fullwid : can be any number of width; default is 76, since ' \' gets added on to most lines. indent : number of spaces to prepend to each line, if desired padpre : number of empty lines to affix beforehand padpost : number of empty lines to affix after text ALIGNASSIGN : flag to line up assignment operators, using longest in list of commands ALLEOL : flag to use the EOL char at the end of all (but the last) lines; if entering separate cmds, turn off. cmdindent : number of spaces to prepend to lines after the [0]th, i.e., just indenting within a single command. Wouldn't be used if there are several sep lines, for example. REP_TIL : replace any tilde '~' with a space at the end (may be useful because the lines are stripped of surrounding white space to start) Returns ------- string : will have the same number of lines as x, but all lines after [0]th will have indentation, and each line except [-1]th will have EOL char at end ''' y = x.split('\n') Nlines = len(y) count = 0 new = [] maxia = 0 # will be loc of assign ops if using ALIGNASSIGN # go through, strip whitespace from ends, and find out where the # '=' are in each line z = [] for line in y: ll = line.strip() if ALIGNASSIGN : # [PT: May 19, 2019] Make this align " = " instead of "=", # because of possible "==" usage that we *don't* want to align if ' = ' in ll: iassign = ll.index(' = ') if iassign > maxia : maxia = iassign z.append(ll) # line up assign ops if asked; also, need nonzero maxia if ALIGNASSIGN and maxia: for i in range(len(z)): # above, we found locus of ' = '; this func places '=' z[i] = padassign(z[i], maxia+1) # go through last time and space everything appropriately for line in z: ll = line.strip() if len(ll) : ll = indent*' ' +ll if count: ll = cmdindent*' ' + ll if ALLEOL : lenll = len(ll) pad = fullwid - lenll if pad > 0: ll+= pad*' ' if REP_TIL: ll = ll.replace('~', ' ') new.append( ll ) count+=1 # put newlines at ends of each, or not if ALLEOL : out = ' \\\n'.join(new) else: out = '\n'.join(new) if padpre: out = padpre*'\n'+out if padpost: out+= padpost*'\n' return out # --------------------------------------------------------------------- def commentize( x, fullwid=76, indent=0, padpre=0, padpost=0, REP_TIL=True): '''Take a string and make it into a comment; indent uniformly, if necessary. Having '||' in a string (surrounded by whitespace) will translate to starting a new line. Inputs ------ x : a string that is the cmd; should arranged into multiple lines (for long commands). Don't need/shouldn't have EOL chars. fullwid : can be any number of width; default is 76, since ' \' gets added on to most lines. indent : number of spaces to prepend to each line, if desired padpre : number of empty lines to affix beforehand padpost : number of empty lines to affix after text REP_TIL : replace any tilde '~' with a space at the end (may be useful because the lines are stripped of surrounding white space to start) Returns ------- string : output >=1 line string version of the original, but spaced to not overrun the fullwid line width. Can be uniformly indented. If the input x is only whitespace or empty, then a null string is returned. ''' y = x.split() new = [] line = indent*' ' + '#' lenstart = len(line) for word in y: if word == "||": new.append(line) line = indent*' ' + '#' elif len(line) + len(word) < fullwid: line+= ' ' + word else: new.append(line) line = indent*' ' + '#' line+= ' ' + word # and get the last line, if there is any text still there beyond # just the starting stuff if len(line) > lenstart : new.append(line) out = '\n'.join(new) if padpre: out = padpre*'\n'+out if padpost: out+= padpost*'\n' if REP_TIL : out = out.replace('~', ' ') return out # --------------------------------------------------------------------- def echoize( x, efile='', indent=0, padpre=0, padpost=0, quote='''"''', REP_TIL=True): '''Take a string and make it into a series of echo statements that will be sent into a text file; indent uniformly, if necessary. Each line is stripped of white space to start, but the '~' character will be replaced by a space at the end, by default. The Inputs ------ x : a string that is the cmd; should arranged into multiple lines (for long commands). Don't need/shouldn't have EOL chars. efile : redirect echo to a file; otherwise, just sent to screen indent : number of spaces to prepend to each line, if desired padpre : number of empty lines to affix beforehand padpost : number of empty lines to affix after text quote : select kind of quote to use for echo; default is ". REP_TIL : replace any tilde '~' with a space at the end (may be useful because the lines are stripped of surrounding white space to start) Returns ------- string : output >=1 line string version of the original, but spaced to not overrun the fullwid line width. Can be uniformly indented. If the input x is only whitespace or empty, then a null string is returned. ''' multix = x.split('\n') N = len(multix) new = [] # set variable to contain output text file name if efile : new.append( '# text for output section in html' ) new.append( 'set otxt = {}'.format(efile) ) for i in range(N): line = multix[i] ll = line.strip() y = indent * ' ' y+= '''echo {}'''.format(quote) + ll + '''{}'''.format(quote) if REP_TIL: y = y.replace('~', ' ') if efile : # choose how to redirect each line red = '> ${otxt}' if ( i == 0 ) else '>> ${otxt}' y+= ''' {}'''.format(red) new.append(y) out = '\n'.join(new) if padpre: out = padpre*'\n'+out if padpost: out+= padpost*'\n' return out # ====================================================================== # ====================================================================== # ===================== specifics for the dsets ======================== # ====================================================================== # ====================================================================== def set_apqc_dirs(ap_ssdict): """Set the names for the QC/ and subdirs for images, etc. in the dict ap_ssdict. These are fixed names, basically. Also include the path location of the abin dir. Parameters ---------- ap_ssdict : dict dictionary of subject uvars Returns ---------- ap_ssdict : dict updated dictionary of subject uvars """ ap_ssdict['odir_qc'] = qcbase + '_' + ap_ssdict['subj'] ap_ssdict['odir_img'] = ap_ssdict['odir_qc'] + '/' + lah.dir_img ap_ssdict['odir_info'] = ap_ssdict['odir_qc'] + '/' + dir_info ap_ssdict['abin_dir'] = get_path_abin() return ap_ssdict def make_apqc_dirs(ap_ssdict, ow_mode='backup', bup_dir=None): """Create the output QC directory, along with its main subdirectories. These names and paths exist in the dictionary ap_ssdict. The mode determines whether this program will overwrite an existing entity of that name or not: shy -> make new QC/ only if one does not already exist overwrite -> purge an existing QC dir and make new QC/ backup -> move an existing QC dir to QC_