PCL: Reading FITS header without loading image
- Thread starter jmurphy
- Start date
The following routine reads all FITS keywords from an existing image file:
This function returns a dynamic array of FITSHeaderKeyword objects. The array will be empty if the file does not have keywords, or if the file format does not support FITS header keywords. Once you have the list of keywords you can iterate the array very easily:
Let me know if this is what you need.
C++:
FITSKeywordArray GetKeywordsFromFile( const String& filePath, const IsoString& inputHints = IsoString() )
{
FileFormat format( File::ExtractExtension( filePath ), true/*read*/, false/*write*/ );
FileFormatInstance file( format );
ImageDescriptionArray images;
if ( !file.Open( images, filePath, inputHints + " verbosity 0" ) )
throw CaughtException();
FITSKeywordArray keywords;
if ( format.CanStoreKeywords() )
file.ReadFITSKeywords( keywords );
if ( !file.Close() )
throw CaughtException();
return keywords;
}This function returns a dynamic array of FITSHeaderKeyword objects. The array will be empty if the file does not have keywords, or if the file format does not support FITS header keywords. Once you have the list of keywords you can iterate the array very easily:
C++:
Console console;
FITSKeywordArray keywords = GetKeywordsFromFile( "/path/to/foo.xisf" );
for ( const FITSHeaderKeyword& k : keywords )
console.WriteLn( "name: " + k.name + " value: " + k.value + " comment: " + k.comment );Let me know if this is what you need.
jmurphy
Well-known member
Thanks Jaun
That was very clearly explained, and it works really well. Very fast.
jmurphy
Well-known member
JavaScript:
// This file is based on FITSKeywords.js which has the following copyright notice:
// ****************************************************************************
// PixInsight JavaScript Runtime API - PJSR Version 1.0
// ****************************************************************************
// FITSKeywords.js - Released 2020/04/17
// ****************************************************************************
//
// This file is part of FITSKeywords Script version 1.0.3
//
// Copyright (C) 2020 Dave Watson.
// Copyright (C) 2009 - 2020 Original author unknown.
// Written by Unknown
// Modified by Dave Watson
//
// Redistribution and use in both source and binary forms, with or without
// modification, is permitted provided that the following conditions are met:
//
// 1. All redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. All redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the names "PixInsight" and "Pleiades Astrophoto", nor the names
// of their contributors, may be used to endorse or promote products derived
// from this software without specific prior written permission. For written
// permission, please contact info@pixinsight.com.
//
// 4. All products derived from this software, in any form whatsoever, must
// reproduce the following acknowledgment in the end-user documentation
// and/or other materials provided with the product:
//
// "This product is based on software from the PixInsight project, developed
// by Pleiades Astrophoto and its contributors (http://pixinsight.com/)."
//
// Alternatively, if that is where third-party acknowledgments normally
// appear, this acknowledgment must be reproduced in the product itself.
//
// THIS SOFTWARE IS PROVIDED BY PLEIADES ASTROPHOTO AND ITS CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PLEIADES ASTROPHOTO OR ITS
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, BUSINESS
// INTERRUPTION; PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; AND LOSS OF USE,
// DATA OR PROFITS) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
// POSSIBILITY OF SUCH DAMAGE.
// ----------------------------------------------------------------------------
/**
*
* @param {String} fitsFilePath
* @returns {FITSKeyword[]}
*/
function LoadFITSKeywords( fitsFilePath ) {
if (!fitsFilePath || !File.exists(fitsFilePath)){
return [];
}
//console.writeln( "File extension: " + getFileExtension(fitsFilePath));
let fileExtension = getFileExtension(fitsFilePath);
if(fileExtension === "xisf") {
// Extract the <xisf> element from file
let f = new File;
f.openForReading( fitsFilePath );
let keywords = new Array;
let rawData = "";
let stringData = "";
let n = 0;
for ( ;; ) { // Load data from file in 1000 byte chunks
rawData = f.read( DataType_ByteArray, 1000 );
if ( f.isEOF )
throw new Error( "Unexpected end of file: " + fitsFilePath );
stringData = stringData + rawData.toString();
n = stringData.search("</xisf>");
if(n >= 0) {break;}
}
// Extract FITSKeywords from <xisf> element
let s = 0;
let e = 0;
for ( ;; ) {
s = stringData.indexOf("<FITSKeyword", s);
if(s < 0) {break;}
s++; // Index past this element start
e = stringData.indexOf("/>", s);
let keywordString = stringData.substring(s, e);
let kname = getKeyword(keywordString,"name");
let kvalue = getKeyword(keywordString,"value");
let kcomment = getKeyword(keywordString,"comment");
keywords.push( new FITSKeyword( kname, kvalue, kcomment ) );
}
f.close();
return keywords;
} else {
let f = new File;
f.openForReading( fitsFilePath );
let keywords = new Array;
for ( ;; ) {
let rawData = f.read( DataType_ByteArray, 80 );
// Parse name part
let name = rawData.toString( 0, 8 );
if ( name.toUpperCase() === "END " ) // end of HDU keyword list?
break;
if ( f.isEOF )
throw new Error( "Unexpected end of file reading FITS keywords, file: " + f.path );
// Parse value / comment parts , handle HIERARCH
let value = "";
let comment = "";
let hasValue = false;
// value separator (an equal sign at byte 8) present?
if ( rawData.at( 8 ) === 61 )
{
// This is a valued keyword
hasValue = true;
// find comment separator slash
let cmtPos = searchCommentSeparator( rawData );
if ( cmtPos < 0 ) // no comment separator?
cmtPos = 80;
// value substring
value = rawData.toString( 9, cmtPos - 9 );
if ( cmtPos < 80 ) // comment substring
comment = rawData.toString( cmtPos + 1, 80 - cmtPos - 1 );
}
else if ( name === 'HIERARCH' )
{
let viPos = searchHierarchValueIndicator( rawData );
if ( viPos > 0 )
{
hasValue = true;
name = rawData.toString( 9, viPos - 10 );
// find comment separator slash
let cmtPos = searchCommentSeparator( rawData );
if ( cmtPos < 0 ) // no comment separator?
cmtPos = 80;
// value substring
value = rawData.toString( viPos + 1, cmtPos - viPos - 1 );
if ( cmtPos < 80 ) // comment substring
comment = rawData.toString( cmtPos + 1, 80 - cmtPos - 1 );
}
}
// If no value in this keyword
if ( !hasValue )
comment = rawData.toString( 8, 80 - 8 ).trim();
// Perform a naive sanity check: a valid FITS file must begin with a SIMPLE=T keyword.
if ( keywords.length === 0 )
if ( name !== "SIMPLE " && value.trim() !== 'T' )
throw new Error( "File does not seem to be a valid FITS file (SIMPLE T not found): " + f.path );
// Create the PJSR FITS keyword and add it to the array.
let fitsKeyWord = new FITSKeyword( name, value, comment );
fitsKeyWord.trim();
keywords.push( fitsKeyWord );
}
f.close();
return keywords;
}
function getFileExtension(filename) {
return filename.slice((filename.lastIndexOf(".") - 1 >>> 0) + 2);
}
function getKeyword(str, keyword) {
let keyLength = (keyword.length + 2);
//console.writeln("[" + str + "] " + keyword + " " + keyLength);
let s = str.indexOf(keyword+'="', 0);
let e = str.indexOf('"', (s + keyLength));
let value = str.substring((s + keyLength), e);
return value;
}
function searchCommentSeparator( b ) {
let inString = false;
for ( let i = 10; i < 80; ++i )
switch ( b.at( i ) )
{
case 39: // single quote
inString ^= true;
break;
case 47: // slash
if ( !inString )
return i;
break;
}
return -1;
}
// In HIERARCH the = sign is after the real keyword name
// Example: HIERARCH LongKeyword = 47.5 / Keyword has > 8 characters, and mixed case
function searchHierarchValueIndicator( b ) {
for ( let i = 9; i < 80; ++i )
switch ( b.at( i ) )
{
case 39: // single quote, = cannot be later
return -1;
case 47: // slash, cannot be later
return -1;
case 61: // =, may be value indicator after all
return i;
}
return -1;
}
return -1;
}
Palmito
Well-known member
Hi John,
Many thanks for sharing the script, it has been very useful to me too.
I encountered an issue, I was in need of the BITPIX keyword from an xisf file but it wasn't pulling it out.
Long story short I suspect a binary sequence in the first bytes to be interpreted as a control sequence and it truncates the first 1000 bytes chunk.
Either read the file in 500 bytes chunks or skip the first 500 bytes by adding the following line just before the loop at line 74:
Cheers and CS !
Many thanks for sharing the script, it has been very useful to me too.
I encountered an issue, I was in need of the BITPIX keyword from an xisf file but it wasn't pulling it out.
Long story short I suspect a binary sequence in the first bytes to be interpreted as a control sequence and it truncates the first 1000 bytes chunk.
Either read the file in 500 bytes chunks or skip the first 500 bytes by adding the following line just before the loop at line 74:
JavaScript:
f.read( DataType_ByteArray, 500 ); // Ignore first 500 bytesCheers and CS !
Hi John,
I would reuse this code in WBPP since I have to do the same job of quickly scanning files to read some FITS keywords during the weights phase.
@Juan Conejero @jmurphy what about promoting this FITSKeywords file as a PJSR standard component?
Well, the original 'unknown' author is the one who subscribes
Yes, this can of course be included as a standard .jsh in include/pjsr. I'll see if this can be done for -10.
Yes, this can of course be included as a standard .jsh in include/pjsr. I'll see if this can be done for -10.
@Juan Conejero @jmurphy I tested this script against the implementation that reads the FITS keywords in WBPP. I found some differences that are worth sharing:
1. the WBPP implementation uses a native PCL implementation to read the keywords, this is significantly faster than the JS version (but OS optimization could contribute significantly since I am reading the same file several times and average the access time)
2. sometimes the two versions read a different number of keywords, this is a red flag. In my tests, the number of keywords read by the PCL implementation is larger than the JS one, sometimes the very first are missing, sometimes few in the middle, etc.
I would stick with the WBPP implementation for now until a final PJSR version will be shared.
For pure testing purposes, I isolated the code of the two implementations and created a testing script, here is the code:
1. the WBPP implementation uses a native PCL implementation to read the keywords, this is significantly faster than the JS version (but OS optimization could contribute significantly since I am reading the same file several times and average the access time)
2. sometimes the two versions read a different number of keywords, this is a red flag. In my tests, the number of keywords read by the PCL implementation is larger than the JS one, sometimes the very first are missing, sometimes few in the middle, etc.
I would stick with the WBPP implementation for now until a final PJSR version will be shared.
For pure testing purposes, I isolated the code of the two implementations and created a testing script, here is the code:
JavaScript:
#include <pjsr/DataType.jsh>
/*
* FITSKeywords Script version 1.0.3
*/
function LoadFITSKeywords(fitsFilePath) {
if (!fitsFilePath || !File.exists(fitsFilePath)) {
return [];
}
//console.writeln( "File extension: " + getFileExtension(fitsFilePath));
let fileExtension = getFileExtension(fitsFilePath);
if (fileExtension === "xisf") {
// Extract the <xisf> element from file
let f = new File;
f.openForReading(fitsFilePath);
let keywords = new Array;
let rawData = "";
let stringData = "";
let n = 0;
for (;;) { // Load data from file in 1000 byte chunks
rawData = f.read(DataType_ByteArray, 1000);
if (f.isEOF)
throw new Error("Unexpected end of file: " + fitsFilePath);
stringData = stringData + rawData.toString();
n = stringData.search("</xisf>");
if (n >= 0) {
break;
}
}
// Extract FITSKeywords from <xisf> element
let s = 0;
let e = 0;
for (;;) {
s = stringData.indexOf("<FITSKeyword", s);
if (s < 0) {
break;
}
s++; // Index past this element start
e = stringData.indexOf("/>", s);
let keywordString = stringData.substring(s, e);
let kname = getKeyword(keywordString, "name");
let kvalue = getKeyword(keywordString, "value");
let kcomment = getKeyword(keywordString, "comment");
keywords.push(new FITSKeyword(kname, kvalue, kcomment));
}
f.close();
return keywords;
} else {
let f = new File;
f.openForReading(fitsFilePath);
let keywords = new Array;
for (;;) {
let rawData = f.read(DataType_ByteArray, 80);
// Parse name part
let name = rawData.toString(0, 8);
if (name.toUpperCase() === "END ") // end of HDU keyword list?
break;
if (f.isEOF)
throw new Error("Unexpected end of file reading FITS keywords, file: " + f.path);
// Parse value / comment parts , handle HIERARCH
let value = "";
let comment = "";
let hasValue = false;
// value separator (an equal sign at byte 8) present?
if (rawData.at(8) === 61) {
// This is a valued keyword
hasValue = true;
// find comment separator slash
let cmtPos = searchCommentSeparator(rawData);
if (cmtPos < 0) // no comment separator?
cmtPos = 80;
// value substring
value = rawData.toString(9, cmtPos - 9);
if (cmtPos < 80) // comment substring
comment = rawData.toString(cmtPos + 1, 80 - cmtPos - 1);
} else if (name === 'HIERARCH') {
let viPos = searchHierarchValueIndicator(rawData);
if (viPos > 0) {
hasValue = true;
name = rawData.toString(9, viPos - 10);
// find comment separator slash
let cmtPos = searchCommentSeparator(rawData);
if (cmtPos < 0) // no comment separator?
cmtPos = 80;
// value substring
value = rawData.toString(viPos + 1, cmtPos - viPos - 1);
if (cmtPos < 80) // comment substring
comment = rawData.toString(cmtPos + 1, 80 - cmtPos - 1);
}
}
// If no value in this keyword
if (!hasValue)
comment = rawData.toString(8, 80 - 8).trim();
// Perform a naive sanity check: a valid FITS file must begin with a SIMPLE=T keyword.
if (keywords.length === 0)
if (name !== "SIMPLE " && value.trim() !== 'T')
throw new Error("File does not seem to be a valid FITS file (SIMPLE T not found): " + f.path);
// Create the PJSR FITS keyword and add it to the array.
let fitsKeyWord = new FITSKeyword(name, value, comment);
fitsKeyWord.trim();
keywords.push(fitsKeyWord);
}
f.close();
return keywords;
}
function getFileExtension(filename) {
return filename.slice((filename.lastIndexOf(".") - 1 >>> 0) + 2);
}
function getKeyword(str, keyword) {
let keyLength = (keyword.length + 2);
//console.writeln("[" + str + "] " + keyword + " " + keyLength);
let s = str.indexOf(keyword + '="', 0);
let e = str.indexOf('"', (s + keyLength));
let value = str.substring((s + keyLength), e);
return value;
}
return -1;
}
/*
* Current WBPP 2.3.0 FITS Header reader implementation
*/
function WBPP_FITSKeywords(filePath) {
let ext = File.extractExtension(filePath).toLowerCase();
let F = new FileFormat(ext, true /*toRead*/ , false /*toWrite*/ );
if (F.isNull) // shouldn't happen
return [];
let f = new FileFormatInstance(F);
if (f.isNull)
return [];
let info = f.open(filePath, "verbosity 0"); // do not fill the console with useless messages
if (!info || (info && info.length <= 0))
return [];
let keywords = [];
if (F.canStoreKeywords)
keywords = f.keywords;
f.close();
return keywords;
}
/*
* helper, note: HISTORY and other multiple keyword names will collide.
*/
function filterAndSortKeywords(keywords) {
return keywords.slice()
.filter(kword => (kword.name.length > 0))
.sort((a, b) => {
return a.name.localeCompare(b.name);
})
.reduce((acc, kword) => {
acc[kword.name] = kword.value;
return acc;
}, {});
}
/* helper */
function keywordNames(keywords) {
return keywords.slice().map(k => k.name);
}
// --------------------------------------------------------
// Main execution
// --------------------------------------------------------
console.show();
let t0;
let keywords;
let elapsed;
let filteredKeyWords;
let names;
// number of reading repetitions
let readAttempts = 10;
let fname = '<your file name>';
// ---------------------------------
// LoadFITSKeywords implementation
// ---------------------------------
t0 = Date.now();
for (let i = 0; i < readAttempts; i++)
keywords = LoadFITSKeywords(fname);
elapsed = (Date.now() - t0) / readAttempts;
names = keywordNames(keywords);
filteredKeyWords = filterAndSortKeywords(keywords);
console.noteln("==== LoadFITSKeywords ====");
console.noteln("number of keywords read: ", keywords.length, "\n");
console.noteln("names: ", JSON.stringify(names, null, 2), "\n");
console.noteln("filtered keyWords: ", JSON.stringify(filteredKeyWords, null, 2), "\n");
console.noteln("in ", elapsed, "ms", "\n");
// ---------------------------------
// WBPP Implementation
// ---------------------------------
t0 = Date.now();
for (let i = 0; i < readAttempts; i++)
keywords = WBPP_FITSKeywords(fname);
elapsed = (Date.now() - t0) / readAttempts;
names = keywordNames(keywords);
filteredKeyWords = filterAndSortKeywords(keywords);
console.noteln("==== WBPP_FITSKeywords ====");
console.noteln("number of keywords read: ", keywords.length, "\n");
console.noteln("names: ", JSON.stringify(names, null, 2), "\n");
console.noteln("filtered keyWords: ", JSON.stringify(filteredKeyWords, null, 2), "\n");
console.noteln("in ", elapsed, "ms");
Last edited:
Just a note of interest on the way we measure execution times. Sorry for the deviation from the main topic, but I'll take the opportunity to point out something important.
We have a core
Besides this, measuring execution times accurately is tricky, especially for relatively short code fragments, because any set of measurements performed for successive executions typically contains large outliers. A non-robust estimator such as the plain mean should not be used for this purpose. This is even more important when there are file I/O operations involved.
A better solution to acquire meaningful execution times is taking the median, or probably better for balanced efficiency, a trimmed mean of a relatively large set of time measurements:
We have a core
ElapsedTime JavaScript object that should always be used for time measurements instead of the Date.now() idiom. For example:
JavaScript:
let T = new ElapsedTime;
for ( let i = 0; i < readAttempts; i++ )
keywords = WBPP_FITSKeywords( fname );
elapsed = T.value / readAttempts;
names = keywordNames( keywords );
filteredKeyWords = filterAndSortKeywords( keywords );
console.noteln( "==== WBPP_FITSKeywords ====" );
console.noteln( "number of keywords read: ", keywords.length, "\n" );
console.noteln( "names: ", JSON.stringify( names, null, 2 ), "\n" );
console.noteln( "filtered keyWords: ", JSON.stringify( filteredKeyWords, null, 2 ), "\n" );
console.noteln( "in ", ElapsedTime.toString( elapsed ) );ElapsedTime is a JavaScript wrapper for our pcl::ElapsedTime C++ class.Besides this, measuring execution times accurately is tricky, especially for relatively short code fragments, because any set of measurements performed for successive executions typically contains large outliers. A non-robust estimator such as the plain mean should not be used for this purpose. This is even more important when there are file I/O operations involved.
A better solution to acquire meaningful execution times is taking the median, or probably better for balanced efficiency, a trimmed mean of a relatively large set of time measurements:
JavaScript:
let times = [];
let T = new ElapsedTime;
for ( let i = 0; i < readAttempts; i++ )
{
T.reset();
keywords = WBPP_FITSKeywords( fname );
times.push( T.value );
}
let trimCount = Math.trunc( 0.15*times.length ); // 15% trimmed mean (for example)
elapsed = Math.trimmedMean( times, trimCount, trimCount );
names = keywordNames( keywords );
filteredKeyWords = filterAndSortKeywords( keywords );
console.noteln( "==== WBPP_FITSKeywords ====" );
console.noteln( "number of keywords read: ", keywords.length, "\n" );
console.noteln( "names: ", JSON.stringify( names, null, 2 ), "\n" );
console.noteln( "filtered keyWords: ", JSON.stringify( filteredKeyWords, null, 2 ), "\n" );
console.noteln( "in ", ElapsedTime.toString( elapsed ) );