Generate XDS.INP: Difference between revisions

(2 intermediate revisions by one other user not shown)

Line 150:

# revision 1.19 . KD 14/10/2023 Pilatus 12M at DLS I23 has reverse phi. Fix download commands.

# revision 1.20 . KD 13/7/2024 Comment out line STRONG_PIXEL=4 in preparation for mid-2024 version of XDS.

REVISION="1.20 (13-~~Jul~~-~~2024~~)"

# revision 1.21 . Thomas Hauß 12/03/2025 define for Bruker sfrm format DET= Bruker-sfrm and corresponding section

# revision 1.22 . KD 28/5/2025 UNTRUSTED_RECTANGLEs for Pilatus3 1M, S/N 10-0159

REVISION="1.22 (28-May-2025)"

#

Line 284:

Line 286:

grep -q MARCONTROL tmp2 && DET=OLDMAR

grep -q "WAVELENGTH=0.0" tmp2 && DET=experimental-ED

if head -c 240 tmp2 | grep -q "FORMAT" && \

head -c 240 tmp2 | grep -q "VERSION" && \

head -c 240 tmp2 | grep -q "HDRBLKS"; then

DET="Bruker-sfrm"

fi

else

h5dump -d "/entry/instrument/detector/description" $FIRSTFRAME | grep -i Eiger > /dev/null && DET=eiger

Line 955:

Line 962:

DIRECTION_OF_DETECTOR_X_AXIS="$R1 0 $R3"

# end of Bruker-cbf section

elif [ "$DET" == "Bruker-sfrm" ]; then

# echo "Detector from Bruker in sfrm format."

# MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT:

MNOPIAS=6

# use complete detector including corners:

TRUSTED_REGION="0 1.42"

SEPMIN=4 # 4 for Pixel Array Detectors

CLUSTER_RADIUS=2 # 2 for Pixel Array Detectors

# to read different format versions read these parametes first:

FORMAT=$(awk -F'FORMAT :' '{print $2}' tmp2 | awk '{print $1}')

VERSION=$(awk -F'VERSION:' '{print $2}' tmp2 | awk '{print $1}')

DETTYPE=$(awk -F'DETTYPE:' '{print $2}' tmp2 | awk '{print $1}')

DETTYPE="BRUKER_$DETTYPE"

echo "Bruker-sfrm file Format: $FORMAT, Version: $VERSION"

# use last value from CCDPARM and convert to integer

OVERLOAD=$(awk -F'CCDPARM:' '{print $2}' tmp2 | awk '{print int($5)}' | head -n1)

DETECTOR="${DETTYPE} MINIMUM_VALID_PIXEL_VALUE=0 OVERLOAD=${OVERLOAD}"

echo "DETECTOR $DETECTOR"

NY=$(awk -F'NROWS :' '{print $2}' tmp2 | awk '{print $1}')

NX=$(awk -F'NCOLS :' '{print $2}' tmp2 | awk '{print $1}')

echo "NX" $NX

echo "NY" $NY

read ORGX ORGY < <(awk -F'CENTER :' '{print $2}' tmp2 | awk '{print $1, $2}')

# There seems to be an error in ORGY, the offset from the detector

# center is calculated wrong with a in inverted sign. Here an attempt

# to correct this, may be not valid in all cases. Found via comparison

# with the Bruker CBF format. There the value of V should be positive!

ORGY=$(echo "scale=2; $NY - $ORGY" | bc -l)

echo "ORGX: $ORGX"

echo "ORGY: $ORGY"

# pixelsize from detector width NX and pixel per cm

pixelpercm=$(awk -F'DETTYPE:' '{print $2}' tmp2 | awk '{print $2}')

if [ NX == 512 ]; then

QX=$(echo "scale=6; 10.0 / $pixelpercm" | bc -l)

QY=QX=$(echo "scale=6; 10.0 / $pixelpercm" | bc -l)

else

QX=$(echo "scale=6; 5.0 / $pixelpercm" | bc -l)

QY=$(echo "scale=6; 5.0 / $pixelpercm" | bc -l)

fi

# QX=0.135 # from Bruker detector specification

# QY=0.135

echo "QX: $QX"

echo "QY: $QY"

# With Version < 11, only one distance written to header, then 2, second value seems ok.

if [ "$VERSION" -lt 11 ]; then

DETECTOR_DISTANCE=$(awk -F'DISTANC:' '{print $2}' tmp2 | awk '{print $1}')

else

DETECTOR_DISTANCE=$(awk -F'DISTANC:' '{print $2}' tmp2 | awk '{print $2}')

fi

DETECTOR_DISTANCE=$(echo "$DETECTOR_DISTANCE * 10" | bc -l) # convert cm to mm

echo "DETECTOR_DISTANCE: $DETECTOR_DISTANCE"

X_RAY_WAVELENGTH=$(awk -F'WAVELEN:' '{print $2}' tmp2 | awk '{print $1}')

echo "X_RAY_WAVELENGTH: $X_RAY_WAVELENGTH"

read TWOTHETA OMEGA PHI KAPPA < <(awk -F'ANGLES :' '{print $2}' tmp2 | awk '{print $1, $2, $3, $4}')

echo "TWOTHETA OMEGA PHI KAPPA: $TWOTHETA $OMEGA $PHI $KAPPA"

# axis 1: 2-theta, 2: omega, 3: phi, 4: chi

axis=$(awk -F'AXIS :' '{print $2}' tmp2 | awk '{print $1}')

if [ "$axis" == 3 ]; then

# PHI scan

echo "PHI scan"

DELTAPHI=$(awk -F'INCREME:' '{print $2}' tmp2 | awk '{print $1}')

R3=$(bc -l <<< "scale=7; s($KAPPA/$DEGTOR) * s($OMEGA/$DEGTOR)")

R1=$(bc -l <<< "scale=7; s($KAPPA/$DEGTOR) * c($OMEGA/$DEGTOR)")

R2=$(bc -l <<< "scale=7; c($KAPPA/$DEGTOR)")

ROTATION_AXIS="$R1 $R2 $R3"

OSCILLATION_RANGE=${DELTAPHI}

STARTING_ANGLE=${PHI}

else

echo "OMEGA scan"

ROTATION_AXIS="0 -1 0"

DELTAOMEGA=$(awk -F'INCREME:' '{print $2}' tmp2 | awk '{print $1}')

OSCILLATION_RANGE=${DELTAOMEGA}

STARTING_ANGLE=${OMEGA}

fi

# detector X-axis

R1=$(bc -l <<< "scale=7; c($TWOTHETA/$DEGTOR)")

R3=$(bc -l <<< "scale=7; s($TWOTHETA/$DEGTOR)")

DIRECTION_OF_DETECTOR_X_AXIS="$R1 0 $R3"

echo "DIRECTION_OF_DETECTOR_X_AXIS: $DIRECTION_OF_DETECTOR_X_AXIS"

echo "STARTING_ANGLE $STARTING_ANGLE"

echo "OSCILLATION_RANGE $OSCILLATION_RANGE"

echo "ROTATION_AXIS $ROTATION_AXIS"

# end of Bruker-sfrm section

else

echo should never come here

Line 1,103:

Line 1,206:

eof

# --------- Append additional detector-specific parameters ----------

if [ "$DET" == "Bruker-cbf" ]; then

if [ "$DET" == "Bruker-cbf" -o "$DET" == "Bruker-sfrm" ]; then

echo "DELPHI=15 ! refine less often than the default of 5" >> XDS.INP

elif [ "$DET" == "adsc-CMOS1" ]; then

Line 1,242:

Line 1,345:

UNTRUSTED_RECTANGLE= 0 4149 3812 3851

eof

# PILATUS 1M, S/N 10-0159

elif [ "$NX" == "981" -a "$NY" == "1043" ]; then

if ! grep -q Flat_field tmp2 ; then

echo Flat_field not found, smaller UNTRUSTED_RECTANGLEs than for bigger Pilatus

cat >> XDS.INP << eof

! the following specifications are for a detector _without_ proper

! flat_field correction; they mask one additional pixel adjacent

! to each UNTRUSTED_RECTANGLE

!EXCLUSION OF VERTICAL DEAD AREAS OF THE PILATUS3 1M, S/N 10-0159

UNTRUSTED_RECTANGLE= 487 495 0 1044

!EXCLUSION OF HORIZONTAL DEAD AREAS OF THE PILATUS3 1M, S/N 10-0159

UNTRUSTED_RECTANGLE= 0 982 195 213

UNTRUSTED_RECTANGLE= 0 982 407 425

UNTRUSTED_RECTANGLE= 0 982 619 637

UNTRUSTED_RECTANGLE= 0 982 831 849

eof

else

echo Flat_field found, smaller UNTRUSTED_RECTANGLEs than for bigger Pilatus

cat >> XDS.INP << eof

!EXCLUSION OF VERTICAL DEAD AREAS OF THE PILATUS3 1M, S/N 10-0159

UNTRUSTED_RECTANGLE= 488 494 0 1044

!EXCLUSION OF HORIZONTAL DEAD AREAS OF THE PILATUS3 1M, S/N 10-0159

UNTRUSTED_RECTANGLE= 0 982 196 212

UNTRUSTED_RECTANGLE= 0 982 408 424

UNTRUSTED_RECTANGLE= 0 982 620 636

UNTRUSTED_RECTANGLE= 0 982 832 848

eof

fi

# Pilatus 12M

elif [ "$NX" == "2463" -a "$NY" == "5071" ]; then

: # handled above, before writing XDS.INP, untrusted regions come from site file

Line 2,240:

Line 2,373:

== Limitations ==

* The script tries to interpret the header of the frames, so is currently limited to Dectris (Pilatus, Eiger), ADSC (Quantum), Rigaku (several types), MAR (CCD and image plate) detectors, and one Bruker detector. Other detectors need some values to be manually filled into XDS.INP - the relevant places are marked with XXX. These are detector properties (type, pixel size and number, min and max counts in a pixel), and experimental parameters like ROTATION_AXIS, OSCILLATION_RANGE, X-RAY_WAVELENGTH, DETECTOR_DISTANCE, and XORG, YORG. For fine-tuning of detector parameters, see the [http://xds.mpimf-heidelberg.mpg.de/html_doc/xds_prepare.html detector-specific templates].

* The script tries to interpret the header of the frames, so is currently limited to Dectris (Pilatus, Eiger), ADSC (Quantum), Rigaku (several types), MAR (CCD and image plate) detectors, and one Bruker detector, and Bruker sfrm Version 18 . Other detectors need some values to be manually filled into XDS.INP - the relevant places are marked with XXX. These are detector properties (type, pixel size and number, min and max counts in a pixel), and experimental parameters like ROTATION_AXIS, OSCILLATION_RANGE, X-RAY_WAVELENGTH, DETECTOR_DISTANCE, and XORG, YORG. For fine-tuning of detector parameters, see the [http://xds.mpimf-heidelberg.mpg.de/html_doc/xds_prepare.html detector-specific templates].

* The authors have made a "best effort" to provide a XDS.INP that results in the correct sign of the anomalous signal. In the case of one detector type (internally called Rigaku SMV) this requires reversal of one detector axis, and a negative DETECTOR_DISTANCE, as is found in some of the [http://xds.mpimf-heidelberg.mpg.de/html_doc/xds_prepare.html detector-specific templates]. '''For an unusual or unknown detector setup, the correct sign of the anomalous signal needs to be established and verified e.g. with a good dataset from a test crystal that has an anomalous signal.''' The authors do not take any responsibility for problems arising from incorrect sign of the anomalous signal, nor - obviously! - for any other mischief arising in or from data processing.

* At some beamlines, the ROTATION_AXIS needs to be -1 0 0 ("backwards") instead of the usual 1 0 0 ("horizontal"), or even 0 1 0 ("vertical") like at one of the PETRA Hamburg BLs. The frame headers do not have this information, but for some beamlines the correct value has been implemented in the script. For other beamlines, the default chosen by [[generate_XDS.INP]] may be wrong and need manual correction of XDS.INP. The correct choice can be enforced by you with a [[Generate_XDS.INP#Site_Files|site file]]. Pls also see the article [[Beamline notes]].

@@ Line 150: / Line 150: @@
 # revision 1.19 . KD 14/10/2023 Pilatus 12M at DLS I23 has reverse phi. Fix download commands.
 # revision 1.20 . KD 13/7/2024 Comment out line STRONG_PIXEL=4 in preparation for mid-2024 version of XDS.
-REVISION="1.20 (13-Jul-2024)"
+# revision 1.21 . Thomas Hauß 12/03/2025 define for Bruker sfrm format DET= Bruker-sfrm and corresponding section
+# revision 1.22 . KD 28/5/2025 UNTRUSTED_RECTANGLEs for Pilatus3 1M, S/N 10-0159
+REVISION="1.22 (28-May-2025)"
 #
@@ Line 284: / Line 286: @@
   grep -q MARCONTROL tmp2 && DET=OLDMAR
   grep -q "WAVELENGTH=0.0" tmp2 && DET=experimental-ED
+ if head -c 240 tmp2 | grep -q "FORMAT" && \
+   head -c 240 tmp2 | grep -q "VERSION" && \
+   head -c 240 tmp2 | grep -q "HDRBLKS"; then
+    DET="Bruker-sfrm"
+ fi
 else
   h5dump -d "/entry/instrument/detector/description" $FIRSTFRAME | grep -i Eiger > /dev/null && DET=eiger
@@ Line 955: / Line 962: @@
    DIRECTION_OF_DETECTOR_X_AXIS="$R1 0 $R3"
 # end of Bruker-cbf section
+elif [ "$DET" == "Bruker-sfrm" ]; then
+# echo "Detector from Bruker in sfrm format."
+  # MINIMUM_NUMBER_OF_PIXELS_IN_A_SPOT:
+  MNOPIAS=6
+  # use complete detector including corners:
+  TRUSTED_REGION="0 1.42"
+  SEPMIN=4            # 4 for Pixel Array Detectors
+  CLUSTER_RADIUS=2    # 2 for Pixel Array Detectors
+  # to read different format versions read these parametes first:
+  FORMAT=$(awk -F'FORMAT :' '{print $2}' tmp2 | awk '{print $1}')
+  VERSION=$(awk -F'VERSION:' '{print $2}' tmp2 | awk '{print $1}')
+  DETTYPE=$(awk -F'DETTYPE:' '{print $2}' tmp2 | awk '{print $1}')
+  DETTYPE="BRUKER_$DETTYPE"
+  echo "Bruker-sfrm file Format: $FORMAT, Version: $VERSION"
+  # use last value from CCDPARM and convert to integer
+  OVERLOAD=$(awk -F'CCDPARM:' '{print $2}' tmp2 | awk '{print int($5)}' | head -n1)
+  DETECTOR="${DETTYPE} MINIMUM_VALID_PIXEL_VALUE=0 OVERLOAD=${OVERLOAD}"
+  echo "DETECTOR $DETECTOR"
+  NY=$(awk -F'NROWS  :' '{print $2}' tmp2 | awk '{print $1}')
+  NX=$(awk -F'NCOLS  :' '{print $2}' tmp2 | awk '{print $1}')
+  echo "NX" $NX
+  echo "NY" $NY
+  read ORGX ORGY < <(awk -F'CENTER :' '{print $2}' tmp2 | awk '{print $1, $2}')
+  # There seems to be an error in ORGY, the offset from the detector
+  # center is calculated wrong with a in inverted sign. Here an attempt
+  # to correct this, may be not valid in all cases. Found via comparison
+  # with the Bruker CBF format. There the value of V should be positive!
+  ORGY=$(echo "scale=2; $NY - $ORGY" | bc -l)
+  echo "ORGX: $ORGX"
+  echo "ORGY: $ORGY"
+  # pixelsize from detector width NX and pixel per cm
+  pixelpercm=$(awk -F'DETTYPE:' '{print $2}' tmp2 | awk '{print $2}')
+  if [ NX == 512 ]; then
+    QX=$(echo "scale=6; 10.0 / $pixelpercm" | bc -l)
+    QY=QX=$(echo "scale=6; 10.0 / $pixelpercm" | bc -l)
+  else
+    QX=$(echo "scale=6; 5.0 / $pixelpercm" | bc -l)
+    QY=$(echo "scale=6; 5.0 / $pixelpercm" | bc -l)
+  fi
+  # QX=0.135  # from Bruker detector specification
+  # QY=0.135
+  echo "QX: $QX"
+  echo "QY: $QY"
+  # With Version < 11, only one distance written to header, then 2, second value seems ok.
+  if [ "$VERSION" -lt 11 ]; then
+    DETECTOR_DISTANCE=$(awk -F'DISTANC:' '{print $2}' tmp2 | awk '{print $1}')
+  else
+    DETECTOR_DISTANCE=$(awk -F'DISTANC:' '{print $2}' tmp2 | awk '{print $2}')
+  fi
+  DETECTOR_DISTANCE=$(echo "$DETECTOR_DISTANCE * 10" | bc -l)  # convert cm to mm
+  echo "DETECTOR_DISTANCE: $DETECTOR_DISTANCE"
+  X_RAY_WAVELENGTH=$(awk -F'WAVELEN:' '{print $2}' tmp2 | awk '{print $1}')
+  echo "X_RAY_WAVELENGTH: $X_RAY_WAVELENGTH"
+  read TWOTHETA OMEGA PHI KAPPA < <(awk -F'ANGLES :' '{print $2}' tmp2 | awk '{print $1, $2, $3, $4}')
+  echo "TWOTHETA OMEGA PHI KAPPA: $TWOTHETA $OMEGA $PHI $KAPPA"
+  # axis 1: 2-theta, 2: omega, 3: phi, 4: chi
+  axis=$(awk -F'AXIS   :' '{print $2}' tmp2 | awk '{print $1}')
+  if [ "$axis" == 3 ]; then
+    # PHI scan
+    echo "PHI scan"
+    DELTAPHI=$(awk -F'INCREME:' '{print $2}' tmp2 | awk '{print $1}')
+    R3=$(bc -l <<< "scale=7; s($KAPPA/$DEGTOR) * s($OMEGA/$DEGTOR)")
+    R1=$(bc -l <<< "scale=7; s($KAPPA/$DEGTOR) * c($OMEGA/$DEGTOR)")
+    R2=$(bc -l <<< "scale=7; c($KAPPA/$DEGTOR)")
+    ROTATION_AXIS="$R1 $R2 $R3"
+    OSCILLATION_RANGE=${DELTAPHI}
+    STARTING_ANGLE=${PHI}
+  else
+    echo "OMEGA scan"
+    ROTATION_AXIS="0 -1 0"
+    DELTAOMEGA=$(awk -F'INCREME:' '{print $2}' tmp2 | awk '{print $1}')
+    OSCILLATION_RANGE=${DELTAOMEGA}
+    STARTING_ANGLE=${OMEGA}
+  fi
+  # detector X-axis
+  R1=$(bc -l <<< "scale=7; c($TWOTHETA/$DEGTOR)")
+  R3=$(bc -l <<< "scale=7; s($TWOTHETA/$DEGTOR)")
+  DIRECTION_OF_DETECTOR_X_AXIS="$R1 0 $R3"
+  echo "DIRECTION_OF_DETECTOR_X_AXIS: $DIRECTION_OF_DETECTOR_X_AXIS"
+  echo "STARTING_ANGLE $STARTING_ANGLE"
+  echo "OSCILLATION_RANGE $OSCILLATION_RANGE"
+  echo "ROTATION_AXIS $ROTATION_AXIS"
+  # end of Bruker-sfrm section
 else
    echo should never come here
@@ Line 1,103: / Line 1,206: @@
 eof
 # --------- Append additional detector-specific parameters ----------
-if [ "$DET" == "Bruker-cbf" ]; then
+if [ "$DET" == "Bruker-cbf" -o "$DET" == "Bruker-sfrm" ]; then
    echo "DELPHI=15 ! refine less often than the default of 5" >> XDS.INP
 elif [ "$DET" == "adsc-CMOS1" ]; then
@@ Line 1,242: / Line 1,345: @@
   UNTRUSTED_RECTANGLE=    0 4149   3812 3851
 eof
+# PILATUS 1M, S/N 10-0159
+  elif [ "$NX" == "981" -a "$NY" == "1043" ]; then
+    if ! grep -q Flat_field tmp2 ; then
+    echo Flat_field not found, smaller UNTRUSTED_RECTANGLEs than for bigger Pilatus
+    cat >> XDS.INP << eof
+! the following specifications are for a detector _without_ proper
+! flat_field correction; they mask one additional pixel adjacent
+! to each UNTRUSTED_RECTANGLE
+!EXCLUSION OF VERTICAL DEAD AREAS OF THE PILATUS3 1M, S/N 10-0159
+UNTRUSTED_RECTANGLE= 487  495     0 1044
+!EXCLUSION OF HORIZONTAL DEAD AREAS OF THE PILATUS3 1M, S/N 10-0159
+UNTRUSTED_RECTANGLE=   0 982   195  213
+UNTRUSTED_RECTANGLE=   0 982   407  425
+UNTRUSTED_RECTANGLE=   0 982   619  637
+UNTRUSTED_RECTANGLE=   0 982   831  849
+eof
+    else
+    echo Flat_field found, smaller UNTRUSTED_RECTANGLEs than for bigger Pilatus
+    cat >> XDS.INP << eof
+!EXCLUSION OF VERTICAL DEAD AREAS OF THE PILATUS3 1M, S/N 10-0159
+UNTRUSTED_RECTANGLE= 488  494     0 1044
+!EXCLUSION OF HORIZONTAL DEAD AREAS OF THE PILATUS3 1M, S/N 10-0159
+UNTRUSTED_RECTANGLE=   0 982   196  212
+UNTRUSTED_RECTANGLE=   0 982   408  424
+UNTRUSTED_RECTANGLE=   0 982   620  636
+UNTRUSTED_RECTANGLE=   0 982   832  848
+eof
+    fi
+# Pilatus 12M
    elif [ "$NX" == "2463" -a "$NY" == "5071" ]; then
      : # handled above, before writing XDS.INP, untrusted regions come from site file
@@ Line 2,240: / Line 2,373: @@
 == Limitations ==
-* The script tries to interpret the header of the frames, so is currently limited to Dectris (Pilatus, Eiger), ADSC (Quantum), Rigaku (several types), MAR (CCD and image plate) detectors, and one Bruker detector. Other detectors need some values to be manually filled into XDS.INP - the relevant places are marked with XXX. These are detector properties (type, pixel size and number, min and max counts in a pixel), and experimental parameters like ROTATION_AXIS, OSCILLATION_RANGE, X-RAY_WAVELENGTH, DETECTOR_DISTANCE, and XORG, YORG. For fine-tuning of detector parameters, see the [http://xds.mpimf-heidelberg.mpg.de/html_doc/xds_prepare.html detector-specific templates].
+* The script tries to interpret the header of the frames, so is currently limited to Dectris (Pilatus, Eiger), ADSC (Quantum), Rigaku (several types), MAR (CCD and image plate) detectors, and one Bruker detector, and Bruker sfrm Version 18 . Other detectors need some values to be manually filled into XDS.INP - the relevant places are marked with XXX. These are detector properties (type, pixel size and number, min and max counts in a pixel), and experimental parameters like ROTATION_AXIS, OSCILLATION_RANGE, X-RAY_WAVELENGTH, DETECTOR_DISTANCE, and XORG, YORG. For fine-tuning of detector parameters, see the [http://xds.mpimf-heidelberg.mpg.de/html_doc/xds_prepare.html detector-specific templates].
 * The authors have made a "best effort" to provide a XDS.INP that results in the correct sign of the anomalous signal. In the case of one detector type (internally called Rigaku SMV) this requires reversal of one detector axis, and a negative DETECTOR_DISTANCE, as is found in some of the [http://xds.mpimf-heidelberg.mpg.de/html_doc/xds_prepare.html detector-specific templates]. '''For an unusual or unknown detector setup, the correct sign of the anomalous signal needs to be established and verified e.g. with a good dataset from a test crystal that has an anomalous signal.''' The authors do not take any responsibility for problems arising from incorrect sign of the anomalous signal, nor - obviously! - for any other mischief arising in or from data processing.
 * At some beamlines, the ROTATION_AXIS needs to be -1 0 0 ("backwards") instead of the usual 1 0 0 ("horizontal"), or even 0 1 0 ("vertical") like at one of the PETRA Hamburg BLs. The frame headers do not have this information, but for some beamlines the correct value has been implemented in the script. For other beamlines, the default chosen by [[generate_XDS.INP]] may be wrong and need manual correction of XDS.INP. The correct choice can be enforced by you with a [[Generate_XDS.INP#Site_Files|site file]]. Pls also see the article [[Beamline notes]].