** TR of 0 != run #1 TR 2.5 in modified afni_proc.py

AFNI version info (afni -ver): Version AFNI_24.0.06 'Caracalla'


I am working on modifying our afni_proc.py script that we use to incorporate all three of our tasks in one afni_proc.py script. This is a continuation from my other post: ** failed to process options... for input files with gen_ss_review_scripts.py - #6 by rickr.

I am using the APMULTI scripts for reference on adding the variables to our afni_proc.py script to loop for each of our three tasks (Alcohol Food VIA). The following is my modified afni_proc.py script without the directory variables and only the Alcohol task set variables. The Food and VIA task will be similar set up as the Alcohol set variables.

# Set input files from Resting State data

set volreg_base_dset = $rs_dir/vr_base_min_outlier+orig.*

set final_volreg_base_dset = $rs_dir/final_epi_vr_base_min_outlier+tlrc.*

set allcostX = $rs_dir/out.allcostX.txt

set basewarp_dset = $rs_dir/mat.basewarp.aff12.1D

set follow_anat_dset = $rs_dir/follow_anat_anat_w_skull+tlrc.*

set anat_final_dset = $rs_dir/anat_final.${subj}+tlrc.*

set anatSS_mat_dset = $rs_dir/anatSS.${subj}_al_junk_mat.aff12.1D

set anatSS_dset = $rs_dir/anatSS.${subj}+orig.*

set anatSS_al_junk_dset = $rs_dir/anatSS.${subj}_al_junk+orig.*

set anatSS_e2a_dset = $rs_dir/anatSS.${subj}_al_junk_e2a_only_mat.aff12.1D

set anatSS_flip_mat_dset = $rs_dir/anatSS.${subj}_flip__al_junk_mat.aff12.1D

set anatSS_flip_dset = $rs_dir/anatSS.${subj}_flip_al_junk+orig.*

set anatSS_unflip_dset = $rs_dir/anatSS.${subj}_unflipped_ob_al_junk_wtal+orig.*

# Set the variables for each task in afni_proc.py 
set tasks = (Alcohol Food VIA)

foreach task ($tasks)
if ("$task" == "Alcohol") then
                 set stim_labels = alcohol neutral alc_novel alc_rep neu_novel neu_rep
                 set sym1 = alcohol
                 set sym2 = alc_novel
                 set sym3 = alc_rep
                 set opp1 = neutral
                 set opp2 = neu_novel
                 set opp3 = neu_rep
                 set label1 = alcoho-neutral
                 set label2 = alc_novel-neu_novel
                 set label3 = alc_rep-neu_rep
                 set task_dir = alc_task
                 set stims = "$stim_dir/alcohol_set_*.1D $stim_dir/neutral_set_*.1D $stim_dir/alc_novel_set_*.1D $stim_dir/alc_rep_set_*.1D $stim_dir/neu_novel_set_*.1D $stim_dir/neu_rep_set_*.1D"
                 set ideals = "sum_ideal_alcohol.1D sum_ideal_neutral.1D sum_ideal_alc_novel.1D sum_ideal_alc_rep.1D sum_ideal_neu_novel.1D sum_ideal_neu_rep.1D"
                 set Block1 = "BLOCK(20,1)"
                 set Block2 = "BLOCK(20,1)"
                 set Block3 = "BLOCK(20,1)"
                 set Block4 = "BLOCK(20,1)"
                 set Block5 = "BLOCK(20,1)"
                 set Block6 = "BLOCK(20,1)"

set top_dir = ${subj_dir}/${task_dir}/${ses}
afni_proc.py -subj_id ${subj}                                                                 \

                -script ${subj}.all.data_prep -scr_overwrite                                  \
                -out_dir ${top_dir}/${task}_task_output                                                                       \
                -copy_anat ${subj_dir}/sswarp_output/anatSS.${subj}.nii                       \
                -anat_has_skull no                                                            \
                -anat_follower anat_w_skull anat ${niix_dir}/*_t1_mprage_tra_p2_iso.nii.gz    \
                -dsets ${niix_dir}/*_${task}_R*.nii.gz                                        \
                           ${anatSS_mat_dset}                                                     \
                           ${anatSS_dset}                                                                                                             \
                           ${anatSS_al_junk_dset}                                                                                                 \
                           ${anatSS_e2a_dset}                                                                                                     \
                           ${anatSS_flip_mat_dset}                                                                                                \
                           ${anatSS_flip_dset}                                                                                                    \
                           ${anatSS_unflip_dset}                                                                                                  \
                       ${volreg_base_dset}                                                    \
                       ${final_volreg_base_dset}                                              \
                           ${basewarp_dset}                                                                                                           \
                           ${follow_anat_dset}                                                                                                    \
                           ${anat_final_dset}                                                                                                     \
                -blocks tcat despike tshift align tlrc volreg blur mask scale regress         \
                -radial_correlate_blocks tcat volreg                                          \
                                -tcat_remove_first_trs    2                                      \
                -align_opts_aea -cost lpc+ZZ -giant_move -check_flip                          \
                -tlrc_base MNI152_2009_template_SSW.nii.gz                                    \
                -tlrc_NL_warp                                                                 \
                -tlrc_NL_warped_dsets ${subj_dir}/sswarp_output/anatQQ.${subj}.nii            \
                        ${subj_dir}/sswarp_output/anatQQ.${subj}.aff12.1D                     \
                        ${subj_dir}/sswarp_output/anatQQ.${subj}_WARP.nii                     \
                -volreg_align_to MIN_OUTLIER                                                  \
                -volreg_align_e2a                                                             \
                -volreg_tlrc_warp                                                             \
                -volreg_warp_dxyz 1.75                                                        \
                -mask_epi_anat yes                                                            \
                -blur_size 6                                                                  \
                -regress_stim_times                                                                                                               \
                                ${stims}                                                                          \
                -regress_stim_labels                                                          \
                        ${stim_labels}                                                                                                        \
                -regress_basis_multi                                                          \
                '${Block1}' '${Block2}' '${Block3}' '${Block4}' '${Block5}' '${Block6}'          \
                -regress_opts_3dD                                                             \
                        -local_times                                                          \
                        -jobs 4                                                               \
                        -gltsym 'SYM: +${sim1} -${opp1}' -glt_label 1 ${label1}               \
                        -gltsym 'SYM: +${sim2} -${opp2}' -glt_label 2 ${label2}                       \
                        -gltsym 'SYM: +${sim3} -${opp3}' -glt_label 3 ${label3}                       \
                -regress_apply_mot_types demean deriv                                         \
                -regress_censor_motion 0.3                                                    \
                -regress_censor_outliers 0.05                                                 \
                -regress_reml_exec                                                            \
                -regress_compute_fitts                                                        \
                -regress_make_ideal_sum sum_ideal.1D                                          \
                -regress_est_blur_epits                                                       \
                -regress_est_blur_errts                                                       \
                -regress_run_clustsim no

The output for this script is:

** warning: removing first 2 TRs from beginning of each run

--> the stimulus timing files must reflect the removal of these TRs

-- template = 'MNI152_2009_template_SSW.nii.gz', exists = 1

-- will use min outlier volume as motion base

-- including default: -find_var_line_blocks tcat

** TR of 0 != run #1 TR 2.5

Is there a setting that I am missing for this issue? Any thoughts on this would be greatly appreciate!

My other question on this script is that if we wanted to use some of the Resting state afni_proc files for this afni_proc.py script such as the ones I have set above as dset variables. Do these input files go into the dset of afni_proc.py or does that need to go somewhere else? I am in Dr. Kyle Simmons lab and we had a discussion about wanting our task processing to use some of the Resting state files since it is the first scan done every time right before the task scans.

I will be happy to give more information on this if needed.

Thank you,


Hi Garrett,

That unclear message refers to the TRs of the -dsets datasets. The error means that there is more than one -dsets parameter, and the TRs do not agree. In this case, I expect you do not intend to input more than one dataset, and the {niix_dir}/*_{task}_R.nii.gz expands to two datasets instead of just one.

The EPI time series datasets are always passed via -dsets, for task or rest. One messy aspect, which is why we use different command files for task and rest, is that most of the -regress options disappear for rest, so having a single script handle those cases might be harder to read.

  • rick

Hi Rick,

Thank you for the reply, with our {niix_dir}/_{task}_R.nii.gz we have one for each run (run1 and run2) that we are wanting to input for each task. So that we can have it run everything for run 1 and also for run 2. Can I set it to have the correct -dset parameter so the TRs correlate with each run?

We are using a different afni_proc.py to run all of our resting state data and then will use this script to run our task data, with the added input files from the resting state afni_proc.py after it is ran.

Thank you,


Oh, then to be sure, what is the output from this? It is complaining about inconsistent TRs.

3dinfo -ntimes -tr {niix_dir}/*_{task}_R.nii.gz

Here is the output for

3dinfo -ntimes -tr {niix_dir}/*_Alcohol_R*.nii.gz:

175 2.500000

175 2.500000

3dinfo -ntimes -tr {niix_dir}/*_Food_R*.nii.gz:

175 2.500000

175 2.500000

3dinfo -ntimes -tr {niix_dir}/*_VIA_R*.nii.gz:

169 2.500000

169 2.500000

Oh my, it did not sink in that -dset contains that whole list of datasets, starting with ${anatSS_mat_dset}. That option should only apply to the EPI time series datasets.

It isn't quite clear to me what the intention of that list is, but all of the lines between (and not including) the -dsets and -blocks lines should probably be removed.

  • rick

Okay thank you! Those are the output files from the resting state afni_proc.py script. It was discussed to have the Alcohol Food and VIA to use some of the output files from the Resting state such as


I wasn't 100% sure where these input files needed to go in the afni_proc.py if we wanted it to use these input files instead of making them again for each task. The thought behind this as that Resting State is always done right before the tasks and these would give us better min_outliers to use or save time on running the script since some of the anat files are already ran for Resting state and should be the exact same for the tasks.

On a separate note, I typically like having options vertically aligned, with their parameters also vertically aligned, and one per line (out to 80 characters). So, this is a version of your afni_proc.py command from above (though leaving out the extra -dsets .. datasets, as discussed above) with that being done, in case you find it useful to read in this format, as well.

afni_proc.py                                                                 \
    -subj_id                  ${subj}                                        \
    -script                   ${subj}.all.data_prep                          \
    -scr_overwrite                                                           \
    -out_dir                  ${top_dir}/${task}_task_output                 \
    -copy_anat                ${subj_dir}/sswarp_output/anatSS.${subj}.nii   \
    -anat_has_skull           no                                             \
    -anat_follower            anat_w_skull anat                              \
                              ${niix_dir}/*_t1_mprage_tra_p2_iso.nii.gz      \
    -dsets                    ${niix_dir}/*_${task}_R*.nii.gz                \
    -blocks                   tcat despike tshift align tlrc volreg blur     \
                              mask scale regress                             \
    -radial_correlate_blocks  tcat volreg                                    \
    -tcat_remove_first_trs    2                                              \
    -align_opts_aea           -cost lpc+ZZ                                   \
                              -giant_move                                    \
                              -check_flip                                    \
    -tlrc_base                MNI152_2009_template_SSW.nii.gz                \
    -tlrc_NL_warp                                                            \
    -tlrc_NL_warped_dsets     ${subj_dir}/sswarp_output/anatQQ.${subj}.nii   \
                              ${subj_dir}/sswarp_output/anatQQ.${subj}.aff12.1D \
                              ${subj_dir}/sswarp_output/anatQQ.${subj}_WARP.nii \
    -volreg_align_to          MIN_OUTLIER                                    \
    -volreg_align_e2a                                                        \
    -volreg_tlrc_warp                                                        \
    -volreg_warp_dxyz         1.75                                           \
    -mask_epi_anat            yes                                            \
    -blur_size                6                                              \
    -regress_stim_times       ${stims}                                       \
    -regress_stim_labels      ${stim_labels}                                 \
    -regress_basis_multi      '${Block1}' '${Block2}' '${Block3}'            \
                              '${Block4}' '${Block5}' '${Block6}'            \
    -regress_opts_3dD         -local_times -jobs 4                           \
                              -gltsym 'SYM: +${sim1} -${opp1}'               \
                              -glt_label 1 ${label1}                         \
                              -gltsym 'SYM: +${sim2} -${opp2}'               \
                              -glt_label 2 ${label2}                         \
                              -gltsym 'SYM: +${sim3} -${opp3}'               \
                              -glt_label 3 ${label3}                         \
    -regress_apply_mot_types  demean deriv                                   \
    -regress_censor_motion    0.3                                            \
    -regress_censor_outliers  0.05                                           \
    -regress_reml_exec                                                       \
    -regress_compute_fitts                                                   \
    -regress_make_ideal_sum   sum_ideal.1D                                   \
    -regress_est_blur_epits                                                  \
    -regress_est_blur_errts                                                  \
    -regress_run_clustsim     no

Two minor comments on the command for having a bit more in the QC HTML:

  • I would change this line
     -radial_correlate_blocks  tcat volreg     \
    to this:
     -radial_correlate_blocks  tcat volreg regress    \
  • I would also add this opt:
     -volreg_compute_tsnr      yes   \


... and also, on a scripting note, it looks like you are using a tcsh syntax for your script, which is certainly fine (and what I use), but then I also see you using some variables like this in your afni_proc.py (AP) command, with single quotes around them:

-gltsym 'SYM: +${sim1} -${opp1}' 

Doesn't that mean that they won't be interpreted, because the $ is not interpreted as a special character within single-quote pairs in tcsh? I would have thought that you would have to have:

-gltsym "SYM: +${sim1} -${opp1}" 

... and similarly throughout the command.

Is that not the case?


Yes you are correct it should have the double quotes around it. That was an oversight by me. Thank you for catching that! I do think that version of afni_proc.py is a little easier to read than the one that I have currently.

Also further on the input files that my PI is wanting to have in the afni_proc.py command. He is wanting to have the following sections provide below from the resting state proc script be used for the task afni_proc.py script that I am working on. So he wants all the output files from these sections in the resting state proc script to be input files for the task afni_proc.py script that I showed in my original post. Is there a good command for afni_proc.py to be able to have it input these files and use them instead of generating new ones?

# catenate outlier counts into a single time series
cat outcount.r*.1D > outcount_rall.1D

# catenate outlier censor files into a single time series
cat rm.out.cen.r*.1D > outcount_${subj}_censor.1D

# get run number and TR index for minimum outlier volume
set minindex = `3dTstat -argmin -prefix - outcount_rall.1D\'`
set ovals = ( `1d_tool.py -set_run_lengths $tr_counts                       \
                          -index_to_run_tr $minindex` )
# save run and TR indices for extraction of vr_base_min_outlier
set minoutrun = $ovals[1]
set minouttr  = $ovals[2]
echo "min outlier: run $minoutrun, TR $minouttr" | tee out.min_outlier.txt

# --------------------------------
# extract volreg registration base
3dbucket -prefix vr_base_min_outlier                           \

# ================================= align ==================================
# for e2a: compute anat alignment transformation to EPI registration base
# (new anat will be current anatSS.4379CCQ+orig)
align_epi_anat.py -anat2epi -anat anatSS.4379CCQ+orig \
       -suffix _al_junk                               \
       -epi vr_base_min_outlier+orig -epi_base 0      \
       -epi_strip 3dAutomask                          \
       -anat_has_skull no                             \
       -cost lpc+ZZ -giant_move -check_flip           \
       -volreg off -tshift off

# warp the volreg base EPI dataset to make a final version
cat_matvec -ONELINE                                                       \
           anatQQ.4379CCQ.aff12.1D                                        \
           anatSS.4379CCQ_al_junk_mat.aff12.1D -I  > mat.basewarp.aff12.1D

3dNwarpApply -master anatQQ.4379CCQ+tlrc -dxyz 2.5                        \
             -source vr_base_min_outlier+orig                             \
             -nwarp "anatQQ.4379CCQ_WARP.nii mat.basewarp.aff12.1D"       \
             -prefix final_epi_vr_base_min_outlier

# create an anat_final dataset, aligned with stats
3dcopy anatQQ.4379CCQ+tlrc anat_final.$subj

# record final registration costs
3dAllineate -base final_epi_vr_base_min_outlier+tlrc -allcostX            \
            -input anat_final.$subj+tlrc |& tee out.allcostX.txt

# -----------------------------------------
# warp anat follower datasets (non-linear)
# warp follower dataset copy_af_anat_w_skull+orig
3dNwarpApply -source copy_af_anat_w_skull+orig                            \
             -master anat_final.$subj+tlrc                                \
             -ainterp wsinc5 -nwarp anatQQ.4379CCQ_WARP.nii               \
             anatQQ.4379CCQ.aff12.1D                                      \
             -prefix follow_anat_anat_w_skull

Hi, Garrett-

I might be missing something, but is there a reason to not just use the same options in the afni_proc.py command in the align, tlrc, and volreg blocks, for both resting state (non-stimulus-timing-based) and task processing?

Running that again should give you essentially the same results. For the same input EPI time series, you should get the same MIN_OUTLIER volume, etc. And you have already calculated the nonlinear warping datasets and are passing those into AP---you would just do the same for the second command.

I don't see a good way to pluck out the guts of one command's proc script to insert them into the other very easily, in general. You could use the MIN_OUTLIER selected volume and provide it as a reference for the second command's volreg block, but I don't see the need here. It seems much cleaner to me to copy the command as the starter for hte second, just adding the task-based aspects to the regress block, essentially. (NB: you might want to not have the despike block in the task-based analysis, esp. if your task is event-related or otherwise quite short.)



So we do have different EPI time series files for resting state, alcohol, food and via. My PI wanted to have all the tasks to use the MIN_OUTLIER files generated by the resting state proc script for the tasks MIN_OUTLIER. This way all the tasks would use the same MIN_OUTLIER and since the resting state scan always done before our task scans it should have the best MIN_OUTLIER to use for our data. The thought behind using the aligned anat files from resting state was to save on time, since those files are already created for the resting state data and we always run the resting state script before the task one.

My PI wanted to see if we could set some of the files generated from the resting state proc script as variables and then input those into the task afni_proc.py script to use for all the tasks. For example when running the task script and its running for the alcohol data it would use the vr_base_min_outlier+orig that was created from the separate resting state data as the min_outlier for alcohol task instead of using the alcohol EPI time series to make a MIN_OUTLIER for the alcohol task. This would be done similarly with the other files made from the align, volreg and tlrc, as well with the food and via task.

I will let @rickr correct me, but I think you could use this for your other AP command cases, then:

-volreg_base_dset  DSET_VR_AP0

to provide the chosen MIN_OUTLIER volreg dataset from your initial AP run. That will be used as a motion reference volume for your EPI-to-EPI alignment with 3dvolreg, and I then I think it will also be used as the volume to align to your anatomical (assuming the -align_epi_ext_dset .. option is not used, as that would take precedence).

That might be the only change you would need to make for your other AP commands. It is true it will take time to re-perform your EPI-anatomical alignment, but that is just computer time rather than human time or tweaking scripts (which could result in scripting errors).


Yes, that resting state vr_base_min_outlier dataset could be passed via -volreg_base_dset to the task analyses. Then they would all use that same registration path. It may repeat the same EPI/anat steps, but the time for that is not important since not only is the anat already skull-stripped, but the extra computation time for NL registration (even though anat->template is done) makes it not worth worrying about.

To be sure, the rest and different tasks are performed without the subject leaving the scanner, is that right?

  • rick

Hi Rick,

Yes all of our scans are done back to back without the participant ever leaving the scanner.

I put in the -volreg_base_dset into my afni_proc.py script and got the following output:

** cannot use -volreg_base_ind or _align_to with _base_dset
   (use sub-brick selection with -volreg_base_dset DSET)
** invalid block : volreg

This is my current script:

set volreg_min_outlier_dset = $rs_dir/vr_base_min_outlier+orig.*

afni_proc.py -subj_id ${subj}                                                            \
             -script ${subj}.${task}.data_prep -scr_overwrite                            \
             -out_dir ${top_dir}/${task}_task_output                                     \
             -copy_anat ${subj_dir}/sswarp_output/anatSS.${subj}.nii                     \
             -anat_has_skull no                                                          \
             -anat_follower anat_w_skull anat ${niix_dir}/*_t1_mprage_tra_p2_iso.nii.gz  \
             -dsets ${niix_dir}/*_${task}_R*.nii.gz                                      \
             -volreg_base_dset "${volreg_min_outlier_dset}"                              \
             -blocks tcat despike tshift align tlrc volreg mask blur scale regress       \
             -radial_correlate_blocks tcat volreg regress                                \
             -tcat_remove_first_trs 2                                                    \
##           -tshift_opts_ts -tpattern alt+z2                                            \
             -align_opts_aea -cost lpc+ZZ -giant_move -check_flip                        \
             -tlrc_base MNI152_2009_template_SSW.nii.gz                                  \
             -tlrc_NL_warp                                                               \
             -tlrc_NL_warped_dsets ${subj_dir}/sswarp_output/anatQQ.${subj}.nii          \
                                   ${subj_dir}/sswarp_output/anatQQ.${subj}.aff12.1D     \
                                   ${subj_dir}/sswarp_output/anatQQ.${subj}_WARP.nii     \
             -volreg_align_to MIN_OUTLIER                                                \
             -volreg_align_e2a                                                           \
             -volreg_tlrc_warp                                                           \
             -volreg_compute_tsnr yes                                                    \
             -volreg_warp_dxyz 1.75                                                      \
             -mask_epi_anat yes                                                          \
             -blur_size 6                                                                \
             -regress_stim_times                                                         \
             	"${stims}"                                                               \
             -regress_stim_labels                                                        \
             	"${stim_labels}"                                                         \
             -regress_basis_multi                                                        \
             	"${Block}"                                                               \
             -regress_opts_3dD -local_times                                              \
                               -jobs 4                                                   \
                               -gltsym "${sym1}" -glt_label 1 ${label1}                  \
                               -gltsym "${sym2}" -glt_label 2 ${label2}                  \
                               -gltsym "${sym3}" -glt_label 3 ${label3}                  \
             -regress_apply_mot_types demean deriv                                       \
#              -test_stim_files no                                                       \
             -regress_censor_motion 0.3                                                  \
             -regress_censor_outliers 0.05                                               \
             -regress_reml_exec                                                          \
             -regress_compute_fitts                                                      \
             -regress_make_ideal_sum sum_ideal.1D                                        \
             -regress_est_blur_epits                                                     \
             -regress_est_blur_errts                                                     \
             -regress_run_clustsim no

Yes, -volreg_base_dset would replace -volreg_align_to MIN_OUTLIER. So only that initial rest run would specify the MIN_OUTLIER, the rest would use the copied -volreg_base_dset.

  • rick

Ah okay thank you!