BgzfFileTypeRecovery.cpp

/*
 *  Copyright (C) 2010  Regents of the University of Michigan
 *
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifdef __ZLIB_AVAILABLE__

#include "BgzfFileTypeRecovery.h"

#include <stdio.h>
#include <stdint.h>

#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <zlib.h>

#include <fstream>
#include <iostream>
#include <stdexcept>
#include <vector>

#pragma pack(push,1)

#define debug false

class RecoveryGzipHeader {
private:
    uint8_t m_ID1;
    uint8_t m_ID2;
    uint8_t m_CM;
    uint8_t m_FLG;
    uint32_t m_MTIME;
    uint8_t m_XFL;
    uint8_t m_OS;
    uint16_t m_XLEN;
public:
    RecoveryGzipHeader() :
        m_ID1(0),
        m_ID2(0),
        m_CM(0),
        m_FLG(0),
        m_MTIME(0),
        m_XFL(0),
        m_OS(0),
        m_XLEN(0)
    {;}

    void defaults() {
        m_ID1 = 31;
        m_ID2 = 139;
        m_CM = 8;
        m_FLG = 4;
        m_MTIME = 0;
        m_XFL = 0;
        m_OS = 255;
        m_XLEN = 6;
    }
    uint8_t ID1() {return m_ID1;}
    uint8_t ID2() {return m_ID2;}
    uint8_t CM() {return m_CM;}
    uint8_t FLG() {return m_FLG;}
    uint32_t MTIME() {return m_MTIME;}
    uint8_t XFL() {return m_XFL;}
    uint8_t OS() {return m_OS;}
    uint16_t XLEN() {return m_XLEN;}
    bool sane() {
        return (m_ID1==31 && m_ID2==139 && m_CM==8 && m_FLG==4 && m_MTIME==0 && m_XFL == 0 && m_OS == 255 && m_XLEN==6);
    }
};

class BGZFHeader : public RecoveryGzipHeader {
private:
    uint8_t m_SI1;
    uint8_t m_SI2;
    uint16_t m_SLEN;    // little endian
    uint16_t m_BSIZE;   // little endian
public:
    BGZFHeader(
            uint8_t m_SI1 = 'B',
            uint8_t m_SI2 = 'C',
            uint16_t m_SLEN = 2,
            uint16_t m_BSIZE = 0
    ) : m_SI1(m_SI1), m_SI2(m_SI2), m_SLEN(m_SLEN), m_BSIZE(m_BSIZE) {;}
    uint8_t SI1() {return m_SI1;}
    uint8_t SI2() {return m_SI2;}
    uint16_t SLEN() {return m_SLEN;}
    uint16_t BSIZE() {return m_BSIZE;}
    bool sane() {
        return RecoveryGzipHeader::sane() && 
            (m_SI1=='B' && m_SI2=='C' && m_SLEN==2 && m_BSIZE > sizeof(BGZFHeader));
    }
};

#pragma pack(pop)

//
// PeekaheadBuffer allows non-destructive peekahead and resyncing
// after read errors when the underlying stream has signatures in the
// data that allow it.
//
// In addition, it has a peek() capability to allow
// the caller to look ahead in the stream to see
// a certain number of bytes before actually consuming them.
//
// The intent is that this class behave as something of a poor
// man's FIFO - with the cost of buffer movement when data is removed.
//
// This is far from ideal, but we basically are moving data around
// when allowing arbitrary peekahead regardless.
//
// The basis for the design is the fact that most read calls to
// various streams at best allow a single character to be peeked
// at, and secondly, do not allow for recovery after an underfling
// framing error occurs.
//
// That is, getchar()/putchar/ungetchar() support a single byte
// peek.  This may be fine for simply parsing applications, but here
// we need to look at blocks up to 64K or more in size to search
// for signatures while re-synchronizing on the underlying stream.
//
class PeekaheadBuffer : public std::vector<uint8_t> {

protected:
    ssize_t m_startPosition;  // start of fresh data

public:
    enum ReturnCode {
        endOfFile = -1,
        reSync = 0,
        ok = 1
    };

    ssize_t startPosition() {return m_startPosition;}

private:
    //
    // when remaining data is 1/8 the size of the full
    // buffer, shift it back down to the start.
    //
    // for use by read(), which will consume data from the buffer.
    //
    void shiftData() {
        if(dataRemaining() < (ssize_t) (std::vector<uint8_t>::size() / 8) ) {
            erase(begin(), begin() + m_startPosition);
            m_startPosition = 0;
        }
    }
    // called when read reports an error for some
    // reason - 
    virtual ReturnCode sync();
public:
    PeekaheadBuffer();
    virtual ~PeekaheadBuffer();

    // return the amount of unused data:
    ssize_t dataRemaining();

    //
    // overload size() to throw an exception - too confusing otherwise
//    size_t size() {abort();}

    //
    // just populate data in buffer from stream - not generic
    //
    // XXX note that it simply ensures that count bytes of data
    // are actually loaded into the buffer - if that amount of
    // data (or more) is present, this call is a NOP.
    //
    virtual ReturnCode readahead(ssize_t count) = 0;

    // read is generic.
    // remove data from our buffer - call non-generic readahead to populate data.
    ReturnCode read(uint8_t *buffer, ssize_t count) {
        ReturnCode rc;

        rc = readahead(count);

        if(rc == ok) {
            uint8_t *src = &(*begin()) + m_startPosition;
            uint8_t *dest = buffer;

            memcpy(dest, src, count);

            m_startPosition += count;  // consume data

            // recover space if wasting too much:
            shiftData();
        } else if(rc == reSync) {
            // peek puked - CRC error, other errors, see if we can sync forwards
            return reSync;
        }  else {
            // failed to get needed data - premature EOF, I guess
            return endOfFile;
        }

        return ok;
    }

};

PeekaheadBuffer::PeekaheadBuffer() : m_startPosition(0)
{
}

PeekaheadBuffer::~PeekaheadBuffer()
{
}

PeekaheadBuffer::ReturnCode PeekaheadBuffer::sync() {
    clear();
    return ok;
}

ssize_t PeekaheadBuffer::dataRemaining()
{
    return std::vector<uint8_t>::size() - m_startPosition;
}


// peekahead buffered file reader class
class FileReader : public PeekaheadBuffer {
    FILE    *m_stream;
public:
    FileReader();
    ~FileReader();
    FileReader(FILE *stream);
    PeekaheadBuffer::ReturnCode readahead(ssize_t count);
    FILE *stream() {return m_stream;}
    bool eof() {return m_stream ? feof(m_stream) : false;}
};

FileReader::FileReader()
{
    m_stream = NULL;
}

FileReader::FileReader(FILE *stream) : m_stream(stream)
{
}

FileReader::~FileReader()
{
    fclose(m_stream);
    m_stream = NULL;
}

//
// fill buffer until we have count bytes of valid
// data.
//
// need to detect error and eof and return appropriate values.
//
PeekaheadBuffer::ReturnCode FileReader::readahead(ssize_t count)
{
    uint8_t buffer[4096];
    while(dataRemaining() < count) {
        int bytesRead = fread(buffer, 1, sizeof(buffer), m_stream);
        if(bytesRead==0) {
            if(ferror(m_stream)) {
                return reSync;
            }
            // ain't getting no more data...
            return endOfFile;
        }
#if 0
        fprintf(stderr, "\n\n");
        int possible = -1;
        for(int i=0;i<bytesRead;i+=16) {
            fprintf(stderr,"%08x: ", i);
            for(int j=0;j<16;j++) {
                if(buffer[i+j]==31 && buffer[i+j+1]==139) {
                    possible = i+j;
                }
                fprintf(stderr,"%02x ", buffer[i+j]);
            }
            fprintf(stderr, "\n");
        }
        if(possible>0) {
            fprintf(stderr,"possible signature at %08x\n", possible);
        }
#endif
        insert(end(), &buffer[0], &buffer[0] + bytesRead);
    }
    return ok;
}

class BGZFReader : public PeekaheadBuffer {
    FileReader  m_fileReader;

public:

    BGZFReader(FILE *stream) : m_fileReader(stream) {;}

    PeekaheadBuffer::ReturnCode readahead(ssize_t count);

    //
    // This will be reading data, and needs to return EOF, etc
    //
    ReturnCode sync() {
        // my internal data is now bad, so we'll scan ahead seeing
        // if we can find a good header
        clear();
        PeekaheadBuffer::ReturnCode rc;
        while((rc = m_fileReader.readahead(sizeof(BGZFHeader)))==ok ) {
            BGZFHeader *header;
            if(rc==endOfFile) return rc;
            // a rc==reSync is ok provided readahead still ensures that header is present
            void *src = &(*(m_fileReader.begin())) + m_fileReader.startPosition();
            header = (BGZFHeader *) src;
            if(header->sane()) {
                if(debug) std::cerr << "BGZFReader::sync returning reSync\n";
                return reSync;  // tell caller they need to sync up
            }
            // consume a byte, then see if we're at a valid block header
            uint8_t throwAwayBuffer;
            rc = m_fileReader.read(&throwAwayBuffer, 1);
        }
        return rc;
    }
    FILE *stream() {return m_fileReader.stream();}

    bool eof() {return dataRemaining()==0 && m_fileReader.eof();}

};

PeekaheadBuffer::ReturnCode BGZFReader::readahead(ssize_t count)
{
    BGZFHeader header;
    // size of inflateBuffer can be determined from ISIZE, I think
    uint8_t inflateBuffer[64*1024];
    uint8_t gzipBuffer[64*1024+1];

    while(dataRemaining() < count) {
        static int loopCount = 0;

        if(debug) std::cerr << "BGZFReader::readahead loopcount = " << loopCount++ << "\n";

        // here we actually read data:
        //  read what should be the header
        //  verify the header
        //  read the remainder of the block
        //  check the CRC validity or perhaps just call unzip
        //
        // XXX the sizeof(header) is wrong:
        PeekaheadBuffer::ReturnCode rc = m_fileReader.read((uint8_t *) (&header), sizeof(header));

        if(rc == endOfFile) {
            return endOfFile;
        }

        // if we have a bad header, start looking forward for a good one,
        if(!header.sane()) {
            // sync does not consume the next good header, it simply syncs()
            // the data stream to the next believed good BGZF header:
            if(debug) std::cerr << "BGZFReader::readahead found corrupt BGZF header - now calling sync()\n";
            rc = sync();
            //
            // even though we can now decompress, we need to tell the caller
            // what is up before they call for more data (caller needs to
            // sync its own record stream):
            return rc;
        }

        // Read the remainder of the block.
        // BSIZE is size of the entire block - 1, so compensate.
        rc = m_fileReader.read((uint8_t *) &gzipBuffer, header.BSIZE() + 1 - sizeof(header));

        if(rc == reSync) {
            if(debug) std::cerr << "BGZFReader::readahead got incomplete BGZF read - now calling sync()\n";
            sync();
            return reSync;
        }

        //
        // we read a header, but our attempt to read more data ended early,
        // so best to just return EOF
        //
        if(rc == endOfFile) {
            return rc;
        }

        PeekaheadBuffer::ReturnCode bgzf_rc = ok;
        // zs.opaque is set when zalloc is NULL
        //
        // NB: zlib inflateInit2() has valgrind errors
        // in versions <1.2.4 - those can be ignored.
        //
        z_stream zs;
        zs.zalloc = NULL;
        zs.zfree = NULL;
        zs.next_in = gzipBuffer;
        zs.avail_in = header.BSIZE() - 16;  // XXX need to check docs for inflate
        zs.next_out = inflateBuffer;
        zs.avail_out = sizeof(inflateBuffer);

        // -15 --> raw inflate - don't look for gzip or zlib header
        // This can be optimized - inflateInit2 does a malloc of
        // approximately 10K (sizeof(inflate_state))
        if(inflateInit2(&zs, -15) != Z_OK) {
            bgzf_rc = reSync;
            if(debug) std::cerr << "BGZFReader::readahead - inflateInit2 failed (out of memory?)\n";
            // XXX fatal?
        }
        if(bgzf_rc==ok && inflate(&zs, Z_FINISH) != Z_STREAM_END) {
            bgzf_rc = reSync;
            if(debug) std::cerr << "BGZFReader::readahead - inflate failed (bad data), calling sync()\n";
        }

        if(bgzf_rc == ok) {
            if(inflateEnd(&zs) == Z_OK) {
                // do something with zs.total_out
                if(debug) std::cout << "hey, got data!  zs.total_out == " << zs.total_out << "\n";

                // append the newly decompressed data
                insert(end(), &inflateBuffer[0], &inflateBuffer[0] + zs.total_out);
            } else {
                // seems exceptionall unlikely, but check this error case too
                bgzf_rc = reSync;
                if(debug) std::cerr << "BGZFReader::readahead - inflateInit2 failed (out of memory?)\n";
                // XXX fatal?
            }
        }

        if(bgzf_rc != ok) {
            inflateEnd(&zs);
            sync();
            return bgzf_rc;
        }

        // may need to get more data - loop back till all is complete
    }

    return ok;

}


#if 0
void testBGZFBuffer()
{
    BGZFReader b(stdin);
    std::vector<uint8_t>::iterator position;
    BGZFReader::ReturnCode rc;

    std::cout << "size = " << b.dataRemaining() << "\n";

    //
    // this should:
    //  decompress a BGZF block, populating the buffer with
    //  unzipped data, possibly returning a BGZFBuffer::ReturnCode of 
    //  resync if it turns out the BGZF data was interrupted by bad
    //  CRC checks.
    //
    rc = b.readahead(64);
    std::cout << "rc = " << rc << " - expect ok (1)\n";
    std::cout << "size (expect 64) = " << b.size() << "\n";
}


int main(int argc, const char **argv)
{
    testBGZFBuffer();
}
#endif



int BgzfFileTypeRecovery::close()
{
    if(bgzfReader) delete bgzfReader;
    bgzfReader = NULL;
    return true;
}


BgzfFileTypeRecovery::BgzfFileTypeRecovery(const char * filename, const char * mode)
{
    if(tolower(mode[0])=='r') {
        FILE *f = fopen(filename,"r");
        bgzfReader = new BGZFReader(f);
    } else {
        // die for now
        if(debug) std::cerr << "Unable to open " << filename << " in mode " << mode << ".\n";
        close();
    }
}

//
// Why is this ever called?
//
bool BgzfFileTypeRecovery::operator == (void * rhs)
{
    throw std::logic_error("BgzfFileTypeRecovery::operator == is dangerous - do not use");
    return false;
}

bool BgzfFileTypeRecovery::operator != (void * rhs)
{
    throw std::logic_error("BgzfFileTypeRecovery::operator != is dangerous - do not use");
    return false;
}

int BgzfFileTypeRecovery::eof()
{
    return bgzfReader->eof();
}

unsigned int BgzfFileTypeRecovery::write(const void * buffer, unsigned int size)
{
    // currently unsupported
    return 0;
}

int BgzfFileTypeRecovery::read(void * buffer, unsigned int size)
{

    if(bgzfReader == NULL) {
        return 0;
    }

    PeekaheadBuffer::ReturnCode rc = bgzfReader->read((uint8_t *) buffer, size);
   //     endOfFile = -1,
//        reSync = 0,
//        ok = 1
    switch(rc) {
        case PeekaheadBuffer::endOfFile:
            // set a flag?
            return 0;
        case PeekaheadBuffer::reSync:
            // we could encode more info in the exception message here:
            if(debug) std::cerr << "throwing BGZF sync exception\n";
            throw std::runtime_error("BGZF stream resync");
        case PeekaheadBuffer::ok:
            //
            // in bgzfReader, we always are ensured we
            // get the full amount of the read, otherwise
            // an error is thrown.
            //
            return size;
    }
    // NOTREACHED
    return 0;
}

int64_t BgzfFileTypeRecovery::tell()
{
    // currently unsupported
    return 0;
}

bool BgzfFileTypeRecovery::seek(int64_t offset, int origin)
{
    // currently unsupported
    return 0;
}


bool BgzfFileTypeRecovery::attemptRecoverySync(bool (*checkSignature)(void *data) , int length)
{
    //
    // creep along a byte at a time, checking for signature.
    //
    // possibly slow.  should only need to scan ahead < 64K bytes
    // or so, however, so should recover in "reasonable" time.
    //
    while( bgzfReader->readahead(length) == PeekaheadBuffer::ok) {
        char ch;
        void *src = &(*(bgzfReader->begin())) + bgzfReader->startPosition();

        //
        // readahead ensures we have 'length' bytes of
        // data to check that is valid in the buffer.
        //
        if((*checkSignature)(src)) return true;
        PeekaheadBuffer::ReturnCode rc  = bgzfReader->read((uint8_t *) &ch, 1);
        if(rc!=PeekaheadBuffer::ok) return false;
        // we consumed a byte, so go back to top of loop,
        // resume filling buffer (if need be) and re-check
    }


    return false;
}

#endif