Generate XDS.INP

This script should be in your $PATH as "generate_XDS.INP" .

#!/bin/bash

# purpose: generate XDS.INP
# revision 0.03 . Kay Diederichs 2/2010 
# revision 0.04 . Kay Diederichs 4/2010 - include alternative ORGX, ORGY calculations for ADSC
# revision 0.05 . Kay Diederichs 5/2010 - grep for "Corrected" in addition to "marccd"; needed for BESSY 
# revision 0.06 . KD 6/2010 - add UNTRUSTED_RECTANGLE and UNTRUSTED_ELLIPSE; use `whereis catmar` and so on 
# revision 0.07 . KD 6/2010 - decide about ORGX/Y info in MAR header being pixels or mm; other fixes
# tested with some datasets from ALS, SSRL, SLS and ESRF; only MARCCD, ADSC/SMV, PILATUS detectors; for other detectors, values must be manually filled in.
#
# usage: e.g. generate_XDS.INP "frms/mydata_1_???.img"
# make sure to have the two quotation marks !
# the ? are wildcards for the frame numbers.
#
# requirement for external programs(s):
# - for MARCCD detectors the "catmar" binary (which can be downloaded from http://www.marresearch.com/download.html) must be in $PATH
# - alternatively, the [[mccd_xdsparams.pl]] script (see http://strucbio.biologie.uni-konstanz.de/xdswiki/index.php/Mccd_xdsparams.pl ) will be used, courtesy of Ana Gonzalez.
# 
# limitations:
# - frame numbers are assumed to start with 1 and run consecutively
# 
# known problems:
# - for ADSC detectors, there are at least three ways to obtain ORGX and ORGY values from the header (see below);
# - the same might be a problem for MAR headers, too (not sure about this)
#
# notes for debugging of the script:
# - add the -v option to the first line, to see where an error occurs
# - comment out the removal of tmp1 and tmp2 in the last line
#
# ====== Start of script ======
if [ "$1" == "help" ] || [ "$1" == "-help" ] || [ "$1" == "-h" ]; then
  echo usage: generate_XDS.INP \"frms/mydata_1_???.img\"   \(_with_ the quotation marks!\)
  exit
fi
echo Full documentation, including complete detector templates and XDS binaries, can be found at
echo http://www.mpimf-heidelberg.mpg.de/~kabsch/xds . More documentation: see XDSwiki
#
# defaults:
#
DETECTOR="XXX MINIMUM_VALID_PIXEL_VALUE=XXX OVERLOAD=XXX"
ORGX=XXX
ORGY=XXX
DETECTOR_DISTANCE=XXX
OSCILLATION_RANGE=XXX
X_RAY_WAVELENGTH=XXX

# see how we are called:
NAME_TEMPLATE_OF_DATA_FRAMES="$1"

# list frames matching the wildcards in NAME_TEMPLATE_OF_DATA_FRAMES
ls -C1 $1 > tmp1 || exit 1

# set upper limit of DATA_RANGE to number of frames (see "limitations" above)
DATA_RANGE=`wc -l tmp1 | awk '{print $1}'`

# set upper limit of SPOT_RANGE to half of DATA_RANGE, but not less than 1
SPOT_RANGE=`echo "scale=0; $DATA_RANGE/2" | bc -l`
SPOT_RANGE=`echo "if ($SPOT_RANGE<1) 1;if ($SPOT_RANGE>1) $SPOT_RANGE" | bc -l`

echo DATA_RANGE=1 $DATA_RANGE

# find out detector type
DET=XXX
strings `head -1 tmp1` | egrep -q 'marccd|Corrected' && DET=mccd
strings `head -1 tmp1` | grep -q PILATUS             && DET=pilatus
strings `head -1 tmp1` | grep -q BEAM_CENTER_X       && DET=adsc
# identify other detector types in the same way (MAR IP would be straightforward)

# parse ASCII header of first frame (except for MARCCD, which therefore requires catmar)

if [ "$DET" == "XXX" ]; then
  echo "this is not a MAR, ADSC/SMV or PILATUS detector - fill in XXX values manually!"
  DETECTOR="XXX MINIMUM_VALID_PIXEL_VALUE=XXX OVERLOAD=XXX"

# find parameters of first frame
elif [ "$DET" == "mccd" ]; then

  DETECTOR="CCDCHESS MINIMUM_VALID_PIXEL_VALUE= 1 OVERLOAD= 65000"
  CATMAR=`whereis -b catmar`
  if [ "$CATMAR" != "catmar:" ]; then

      # inspect frame header and get rid of blanks:
      catmar `head -1 tmp1` | sed s/\ //g > tmp2

      # find X_RAY_WAVELENGTH:
      X_RAY_WAVELENGTH=`grep Wavelength tmp2 | sed s/Wavelength.Ang]=//`

      # find QX, ORGX and ORGY:
      QX=`grep Pixelsizex tmp2 | sed s/Pixelsizex=//`
      QX=`echo "scale=10; $QX/1000000" |bc -l `

      NX=`grep pixelsin1line tmp2 | sed s/#pixelsin1line=//`

      ORGX=`grep Centerx tmp2 | sed s/Centerx=//`
      ORGX=`echo "scale=1; ($ORGX+50)/1000" | bc -l `

      ORGY=`grep Centery tmp2 | sed s/Centery=//`
      ORGY=`echo "scale=1; ($ORGY+50)/1000" | bc -l `

      # find DETECTOR_DISTANCE and OSCILLATION_RANGE:
      DETECTOR_DISTANCE=`grep Distance tmp2 | sed s/Distance.mm]=//`
      STARTING_PHI=`grep StartingPHI tmp2 | sed s/StartingPHI=//`
      ENDING_PHI=`grep EndingPHI tmp2 | sed s/EndingPHI=//`
      OSCILLATION_RANGE=`echo "scale=3; ($ENDING_PHI-$STARTING_PHI)/1000" | bc -l`

  else
      CATMAR=`whereis -b mccd_xdsparams.pl`
      if [ "$CATMAR" != "mccd_xdsparams:" ]; then
        mccd_xdsparams.pl -v `head -1 tmp1` > tmp2
        NX=`awk '/NX/{print $2}' tmp2`
        QX=`awk '/QX/{print $6}' tmp2`
        X_RAY_WAVELENGTH=`awk '/X-RAY_WAVELENGTH/{print $2}' tmp2`
        DETECTOR_DISTANCE=`awk '/DETECTOR_DISTANCE/{print $2}' tmp2`
        ORGX=`awk '/ORGX/{print $2}' tmp2`
        ORGY=`awk '/ORGY/{print $4}' tmp2`
        OSCILLATION_RANGE=`awk '/OSCILLATION_RANGE/{print $2}' tmp2`
      else
        echo "could not find catmar or mccd_xdsparams.pl - fill XXX values manually!"
      fi
  fi
# at most BLs, ORGX and ORGY are in pixels, but sometimes in mm ... guess:
  NXBYFOUR=`echo "scale=0; $NX/4" | bc -l `
  ORGXINT=`echo "scale=0; $ORGX/1" | bc -l `
  if [ $ORGXINT -lt $NXBYFOUR ]; then
#    echo ORGX ORGY QX = $ORGX $ORGY $QX
     ORGX=`echo "scale=1; $ORGX/$QX" | bc -l`
     ORGY=`echo "scale=1; $ORGY/$QX" | bc -l`
     echo MARCCD detector: header ORGX, ORGY seem to be in mm ... converting to pixels
  else
     echo MARCCD detector: header ORGX, ORGY seem to be in pixel units
  fi

elif [ "$DET" == "adsc" ]; then

  DETECTOR="ADSC MINIMUM_VALID_PIXEL_VALUE= 1 OVERLOAD= 65000"
  echo this is an ADSC detector. Obtaining ORGX, ORGY from the header depends on beamline setup.
  strings `head -1 tmp1` | sed s/\;// > tmp2

      # find X_RAY_WAVELENGTH:
      X_RAY_WAVELENGTH=`grep WAVELENGTH tmp2 | sed s/WAVELENGTH=//`

      # find NX, QX, ORGX and ORGY:
      NX=`grep SIZE1 tmp2 | tail -1 | sed s/SIZE1=//`
      QX=`grep PIXEL_SIZE tmp2 | sed s/PIXEL_SIZE=//`
      BEAM_CENTER_X=`grep BEAM_CENTER_X tmp2 | sed s/BEAM_CENTER_X=//`
      BEAM_CENTER_Y=`grep BEAM_CENTER_Y tmp2 | sed s/BEAM_CENTER_Y=//`
# fix 2010-04-26 - tell user about possible ORGX, ORGY alternatives -  
# at ESRF and ... (pls fill in!) the following should be used:
      ORGX=`echo "scale=1; $BEAM_CENTER_Y/$QX" | bc -l `
      ORGY=`echo "scale=1; $BEAM_CENTER_X/$QX" | bc -l `
      echo ATTENTION: at ESRF BLs use: ORGX=$ORGX ORGY=$ORGY 
# this 2nd alternative convention should be used at the following beamlines (pls complete the list): ALS 5.0.3, ...
      ORGX=`echo "scale=1; $NX-$BEAM_CENTER_X/$QX" | bc -l `
      ORGY=`echo "scale=1; $BEAM_CENTER_Y/$QX" | bc -l `
      echo ATTENTION: at e.g. ALS 5.0.3 use: ORGX=$ORGX ORGY=$ORGY 
# this 3rd alternative convention should be used at the following beamlines (pls complete the list): ALS 8.2.2, ... 
      ORGX=`echo "scale=1; $BEAM_CENTER_X/$QX" | bc -l `
      ORGY=`echo "scale=1; $NX-$BEAM_CENTER_Y/$QX" | bc -l `
      echo ATTENTION: at e.g. ALS 8.2.2 use: ORGX=$ORGX ORGY=$ORGY - this is now written to XDS.INP
# the latter alternative is written into the generated XDS.INP ! You have to correct this manually in XDS.INP, or adjust this script.
      # find DETECTOR_DISTANCE and OSCILLATION_RANGE:
      DETECTOR_DISTANCE=`grep DISTANCE tmp2 | sed s/DISTANCE=//`
      OSCILLATION_RANGE=`grep OSC_RANGE tmp2 | sed s/OSC_RANGE=//`

elif [ "$DET" == "pilatus" ]; then
  DETECTOR="DETECTOR=PILATUS  MINIMUM_VALID_PIXEL_VALUE=0  OVERLOAD= 1048576"
  echo this is a Pilatus detector
  head -50 `head -1 tmp1` | sed s/#//> tmp2

      # find X_RAY_WAVELENGTH:
      X_RAY_WAVELENGTH=`grep Wavelength tmp2 | sed -e s/Wavelength// -e s/A// | awk '{print $1}'`

      # find ORGX and ORGY:
      ORGX=`grep Beam_xy tmp2 | sed -e s/\(// -e s/\)// -e s/\,// | awk '{print $2}'`
      ORGY=`grep Beam_xy tmp2 | sed -e s/\(// -e s/\)// -e s/\,// | awk '{print $3}'`

      # find DETECTOR_DISTANCE and OSCILLATION_RANGE:
      DETECTOR_DISTANCE=`awk '/distance/{print $2}' tmp2`
      DETECTOR_DISTANCE=`echo "$DETECTOR_DISTANCE*1000" | bc -l`

      OSCILLATION_RANGE=`awk '/Angle/{print $2}' tmp2`

else
  echo should never come here
  exit 1
fi

echo ORGX= $ORGX ORGY= $ORGY - check these values with adxv !
echo DETECTOR_DISTANCE= $DETECTOR_DISTANCE
echo OSCILLATION_RANGE= $OSCILLATION_RANGE
echo X-RAY_WAVELENGTH= $X_RAY_WAVELENGTH

# now we know everything that is required to generate XDS.INP

cat > XDS.INP << eof
JOB= XYCORR INIT COLSPOT IDXREF DEFPIX INTEGRATE CORRECT
ORGX= $ORGX ORGY= $ORGY  ! check these values with adxv !
DETECTOR_DISTANCE= $DETECTOR_DISTANCE
OSCILLATION_RANGE= $OSCILLATION_RANGE
X-RAY_WAVELENGTH= $X_RAY_WAVELENGTH
NAME_TEMPLATE_OF_DATA_FRAMES=$NAME_TEMPLATE_OF_DATA_FRAMES
! REFERENCE_DATA_SET=xxx/XDS_ASCII.HKL ! e.g. to ensure consistent indexing  
DATA_RANGE=1 $DATA_RANGE
SPOT_RANGE=1 $SPOT_RANGE
! BACKGROUND_RANGE=1 10 ! rather use defaults (first 5 degree of rotation)

SPACE_GROUP_NUMBER=0                   ! 0 if unknown
UNIT_CELL_CONSTANTS= 70 80 90 90 90 90 ! put correct values if known
INCLUDE_RESOLUTION_RANGE=50 0  ! after CORRECT, insert high resol limit; re-run CORRECT

FRIEDEL'S_LAW=FALSE     ! This acts only on the CORRECT step
! If the anom signal turns out to be, or is known to be, very low or absent,
! use FRIEDEL'S_LAW=TRUE instead (or comment out the line); re-run CORRECT

! remove the "!" in the following line:
! STRICT_ABSORPTION_CORRECTION=TRUE
! if the anomalous signal is strong: in that case, in CORRECT.LP the three
! "CHI^2-VALUE OF FIT OF CORRECTION FACTORS" values are significantly> 1, e.g. 1.5
!
! exclude (mask) untrusted areas of detector, e.g. beamstop shadow :
! UNTRUSTED_RECTANGLE= 1800 1950 2100 2150 ! x-min x-max y-min y-max ! repeat
! UNTRUSTED_ELLIPSE= 2034 2070 1850 2240 ! x-min x-max y-min y-max ! if needed
!
! parameters with changes wrt default values:
TRUSTED_REGION=0.00 1.2  ! partially use corners of detectors; 1.41421=full use
VALUE_RANGE_FOR_TRUSTED_DETECTOR_PIXELS=7000. 30000. ! often 8000 is ok
MINIMUM_ZETA=0.05        ! integrate close to the Lorentz zone; 0.15 is default
STRONG_PIXEL=6           ! COLSPOT: only use strong reflections (default is 3)
REFINE(INTEGRATE)=CELL BEAM ORIENTATION ! AXIS DISTANCE 

! parameters specifically for this detector and beamline:
DETECTOR= $DETECTOR
!NX= 3072 NY= 3072  QX= 0.10260  QY= 0.10260 ! XDS finds this out by itself
DIRECTION_OF_DETECTOR_X-AXIS=1 0 0
DIRECTION_OF_DETECTOR_Y-AXIS=0 1 0
INCIDENT_BEAM_DIRECTION=0 0 1
ROTATION_AXIS=1 0 0    ! at e.g. SERCAT ID-22 this needs to be -1 0 0
FRACTION_OF_POLARIZATION=0.98   ! better value is provided by beamline staff!
POLARIZATION_PLANE_NORMAL=0 1 0
eof

echo XDS.INP is ready for use. The file has only the most important keywords.
echo After running xds, inspect at least BKGPIX.cbf and FRAME.cbf with XDS-Viewer!
rm tmp1 tmp2

For MarCCD data, generate_XDS.INP relies on mccd_xdsparams.pl which is below, and should also be in your $PATH . Alternatively, catmar (which can be downloaded from http://www.marresearch.com/download.html) may be used.

#!/usr//bin/perl -w
#
#
#                        Copyright 2004
#                              by
#                 The Board of Trustees of the
#               Leland Stanford Junior University
#                      All rights reserved.
#
#                       Disclaimer Notice
#
#     The items furnished herewith were developed under the sponsorship
# of the U.S. Government.  Neither the U.S., nor the U.S. D.O.E., nor the
# Leland Stanford Junior University, nor their employees, makes any war-
# ranty, express or implied, or assumes any liability or responsibility
# for accuracy, completeness or usefulness of any information, apparatus,
# product or process disclosed, or represents that its use will not in-
# fringe privately-owned rights.  Mention of any product, its manufactur-
# er, or suppliers shall not, nor is it intended to, imply approval, dis-
# approval, or fitness for any particular use.  The U.S. and the Univer-
# sity at all times retain the right to use and disseminate the furnished
# items for any purpose whatsoever.
#
#                       Permission Notice
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTA-
# BILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO
# EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR
# THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
# MARCDD image header format
#
# TIFF header 1024 bytes
# Private frame header 3072 bytes
## file/header format parameters 256 bytes
## data statistics 128 bytes
## more statistics 256 bytes
## goniostat parameters 128 bytes
### xtal_to_detector int32 (mm*1000)
### beam_x int32 (pixel*1000)
### beam_y int32 (pixel*1000)
## detector parameters 128 bytes
### detector_type int32
### pixelsize_x int32 (nanometers
### pixelsize_y int32 (nanometers
##X-ray source parameters 128 bytes

die "ERROR: Missing Argument. \nUsage: $0 [options] <image_filename>\n" unless @ARGV;

while ($_ = $ARGV[0], /^-/) {
        shift;
        last if /^--$/;
        # if (/^-D(.*)/) { $debug = $1 }
        if (/^-v/)     { $verbose = 1 }
        # ...           # other switches
}

open ( FILE, '<', $ARGV[0] ) or
        die ( "ERROR: Cannot open file" );

seek ( FILE, 1024+80, 0 );
read ( FILE , $size_fast, 4 );
read ( FILE , $size_slow, 4 );

seek ( FILE, 1024+256+128+256, 0 );

read ( FILE , $xtal_to_detector, 4 );
read ( FILE , $beam_x, 4 );
read ( FILE , $beam_y, 4 );

seek ( FILE, 1024+256+128+256+16, 0);

read ( FILE , $exp, 4 ); #exp time x 1000

seek ( FILE, 1024+256+128+256+44, 0);
read ( FILE , $start_phi, 4 ); #start phi x 1000
seek ( FILE, 1024+256+128+256+76, 0);
read ( FILE , $end_phi, 4 ); #end phi x 1000

seek ( FILE, 1024+256+128+256+128, 0 );

read ( FILE , $detector_type, 4 );
read ( FILE , $pixelsize_x, 4 );
read ( FILE , $pixelsize_y, 4 );

seek ( FILE, 1024+256+128+256+128+128+12, 0 );
read ( FILE , $wavelength, 4 ); #wavelength x 10000

close ( FILE );

$size_fast = unpack ('i*' ,$size_fast);
$size_slow = unpack ('i*' ,$size_slow);
$xtal_to_detector =unpack('i*' ,$xtal_to_detector) / 1000;
$detector_type =unpack('i*' ,$detector_type);
$beam_x = unpack ('i*' ,$beam_x) / 1000 ; # pixels
$beam_y = unpack ('i*' ,$beam_y) / 1000 ; # pixels
$beam_xc = $size_fast - $beam_x ;
$beam_yc = $size_fast - $beam_y ;
$pixelsize_x = unpack ('i*' ,$pixelsize_x) / 1000000 ;  # mm
$pixelsize_y = unpack ('i*' ,$pixelsize_y) / 1000000 ;  # mm
$beam_x_mm = $beam_x * $pixelsize_x  ; # mm
$beam_y_mm = $beam_y * $pixelsize_y ; # mm
$exp = unpack ('i*' ,$exp) / 1000 ; #seconds
$start_phi = unpack ('i*' ,$start_phi); #mdeg
$end_phi = unpack ('i*' ,$end_phi); #mdeg
$osc = ($end_phi - $start_phi)/1000; # deg
$wavelength = unpack ('i*' ,$wavelength)/100000  ; #A

if ($verbose) {
        print "NX= ${size_fast} NY= ${size_slow} QX= ${pixelsize_x} QY= ${pixelsize_y} \n";
        print "DETECTOR_DISTANCE= ${xtal_to_detector} \n";
        print "ORGX= ${beam_xc}  ORGY= ${beam_yc} \n";
        print "OSCILLATION_RANGE= ${osc} \n";
        print "X-RAY_WAVELENGTH= ${wavelength} \n";

} else {
        print "$size_fast,$size_slow", "\n";
        print "$xtal_to_detector", "\n";
        print "$beam_x_mm,$beam_y_mm", "\n";
        print "$pixelsize_x", "\n";
        print "$exp","\n";
        print "$osc","\n";
        print "$wavelength","\n";
}

exit