MemoryMapArray.h

/*
 *  Copyright (C) 2010  Regents of the University of Michigan
 *
 *   This program is free software: you can redistribute it and/or modify
 *   it under the terms of the GNU General Public License as published by
 *   the Free Software Foundation, either version 3 of the License, or
 *   (at your option) any later version.
 *
 *   This program is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 *   You should have received a copy of the GNU General Public License
 *   along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef __MEMORYMAPARRAY_H
#define __MEMORYMAPARRAY_H

#ifndef __STDC_LIMIT_MACROS
#define __STDC_LIMIT_MACROS
#endif
#include <errno.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <string>
#include <sys/types.h>
#include <time.h>

// STL:
#include <ostream>
#include <sstream>

#include "MemoryMap.h"


//
// This file defines a template for generating memory map backed arrays
// of different types of values.
//
// The template assumes that the mapped files are broken into two parts,
// first, a header (MemoryMapArrayHeader), then followed by the data
// in the array.
//
// typedefs are used to declare various types of arrays beforehand,
// since there will be only a few.
//
// They are:
//      mmapArrayUint32_t;
//      mmapArrayBool_t;
//      mmapArray4Bit_t;
//
// XXX consider adding env("USER"), argv[0], date/time creation, etc.
//
class MemoryMapArrayHeader
{
public:
    void constructorClear()
    {
        memset(this, 0, sizeof(*this));
    }
    uint32_t     typeCookie;
    uint32_t     typeVersion;
    uint32_t     contentCookie;
    uint32_t     contentVersion;
    size_t       headerSize;

    // file generation info
    time_t      creationDate;
    char        creationUser[32];
    char        creationHost[32];
    char        application[32];
    // now describe the data:
    size_t      elementCount;
    void debugPrint(FILE *);
    size_t getHeaderSize(int i)
    {
        return sizeof(*this);
    }

    void setApplication(const char *s)
    {
        strncpy(application, s, sizeof(application)-1);
    }
    void setCreationUser(const char *s)
    {
        strncpy(creationUser, s, sizeof(creationUser)-1);
    }
    void setCreationHost(const char *s)
    {
        strncpy(creationHost, s, sizeof(creationHost)-1);
    }
};

//
// stream output for header information
//
std::ostream &operator << (std::ostream &stream, MemoryMapArrayHeader &h);

//
// This class object represents the application specific information that doesn't
// fit in the general header above.  Since it is only allocated via an mmap operation,
// as part of the mapped file, the destructor must never be called.  The virtual
// destructor is declared to eliminate gcc warnings.
//
// For many arrays, this will be empty.
//
struct MemoryMapGenericHeader
{
protected:
    size_t  headerSize;     // set in ::create and ::open only
public:
    size_t getHeaderSize()
    {
        return headerSize;
    }
    // other stuff follows...
};

template <
class elementT,
typename indexT,
unsigned int cookieVal,
unsigned int versionVal,
elementT accessorFunc(void *base, indexT),
void setterFunc(void *base, indexT, elementT),
size_t elementCount2BytesFunc(indexT),
class arrayHeaderClass>
class MemoryMapArray : public MemoryMap
{
protected:
    arrayHeaderClass    *header;
    void                *data;
    std::string         errorStr;
public:
    void constructorClear()
    {
        header = NULL;
        data = NULL;
//      errorStr = "";
    }
    MemoryMapArray()
    {
        constructorClear();
    }
    ~MemoryMapArray()
    {
        if (data) close();
    }

    const std::string &getErrorString()
    {
        return errorStr;
    }

    arrayHeaderClass &getHeader()
    {
        return *header;
    }

    void setContentCookie(uint32_t c)
    {
        header->contentCookie = c;
    }
    void setContentVersion(uint32_t v)
    {
        header->contentVersion = v;
    }

    // accessing
    inline elementT operator[](indexT i)
    {
        return accessorFunc(data, i);
    }
    inline void set(indexT i, elementT v)
    {
        setterFunc(data, i, v);
    }



    /// Create a vector with elementCount memebers.
    //
    /// Does administrative setup of the header and populating this
    /// class members.  User will need to finish populating the
    /// contents of the metaData and data sections.
    ///
    /// If file==NULL, the underlying allocation is done via malloc(),
    /// so that the results of write access to this vecor are not
    /// saved in a file.
    ///
    /// If file!=NULL, a file will be created on disk, and all
    /// write accesses done via the method ::set will be persistent
    /// in that file.
    ///
    int create(const char *file, indexT elementCount, int optionalHeaderCount = 0)
    {
        size_t len = elementCount2BytesFunc(elementCount) +
                     header->getHeaderSize(optionalHeaderCount);
        int rc;
        rc = MemoryMap::create(file, len);
        if (rc)
        {
            std::ostringstream buf;
            buf << file << ": failed to create file";
            errorStr = buf.str();
            close();
            return rc;
        }
        header = (arrayHeaderClass *) MemoryMap::data;
        header->constructorClear();
        header->typeCookie = cookieVal;
        header->typeVersion = versionVal;
        header->headerSize = header->getHeaderSize(optionalHeaderCount);
        header->elementCount = elementCount;
        data = (elementT *)((char *) MemoryMap::data + header->headerSize);

        const char *env;
        char hostname[256];
        env = getenv("USER");
        if (env) header->setCreationUser(env);
        header->creationDate = time(NULL);
#if defined(WIN32)
        hostname[0] = '\0';
#else
        gethostname(hostname, sizeof(hostname));
#endif
        header->setCreationHost(hostname);
        return 0;
    }

    /// allow anonymous (malloc) create.
    ///
    /// we do this when we don't expect to save the results.
    ///
    /// The single use case so far is in GenomeSequence::populateDBSNP.
    ///
    int create(indexT elementCount, int optionalHeaderCount = 0)
    {
        return create(NULL, elementCount, optionalHeaderCount);
    }

    //
    // Open the given filename.  flags may be set to
    // O_RDONLY or O_RDWR, and allows the file to be
    // condtionally written to.
    //
    // Several sanity checks are done:
    //   compare the expected cookie value to the actual one
    //   compare the expected version value to the actual one
    //
    // if either condition is not met, the member errorStr is
    // set to explain why, and true is returned.
    //
    // If there were no errors, false is returned.
    //
    bool open(const char *file, int flags = O_RDONLY)
    {
        int rc = MemoryMap::open(file, flags);
        if (rc)
        {
            std::ostringstream buf;
            buf << file << ": open() failed (error=" << strerror(errno) << ").";
            errorStr = buf.str();
            return true;
        }
        header = (arrayHeaderClass *) MemoryMap::data;
        data = (elementT *)((char *) MemoryMap::data + header->headerSize);
        if (header->typeCookie!=cookieVal)
        {
            std::ostringstream buf;
            buf << file << ": wrong type of file (expected type "
            << cookieVal << " but got " << header->typeCookie << ")";
            errorStr = buf.str();
            // XXX insert better error handling
            close();
            return true;
        }
        if (header->typeVersion!=versionVal)
        {
            std::ostringstream buf;
            buf << file << ": wrong version of file (expected version "
            << versionVal << " but got " << header->typeVersion << ")";
            errorStr = buf.str();
            // XXX insert better error handling
            close();
            return true;
        }
        return false;
    }

    void close()
    {
        constructorClear();
        MemoryMap::close();
    }
    void debugPrint(FILE *f)
    {
        if (header) header->debugPrint(f);
    }

    size_t getElementCount()
    {
        return header->elementCount;
    }

};

struct emptyGenericHeader : public MemoryMapGenericHeader
{
public:
    size_t getHeaderSize()
    {
        return sizeof(*this);
    }
};

//
// define the uint32 array type:
//
inline uint32_t mmapUint32Access(void *base, uint32_t index)
{
    return ((uint32_t *)base)[index];
}
inline void mmapUint32Set(void *base, uint32_t index, uint32_t v)
{
    ((uint32_t *)base)[index] = v;
}
inline size_t mmapUint32elementCount2Bytes(uint32_t i)
{
    return sizeof(uint32_t) * i;
}

typedef MemoryMapArray<
uint32_t,
uint32_t,
0x16b3816c,
20090109,
mmapUint32Access,
mmapUint32Set,
mmapUint32elementCount2Bytes,
MemoryMapArrayHeader
> mmapArrayUint32_t;

//
// define the boolean memory mapped array type.
// NB: it is limited to 2**32 elements
//
inline bool mmapBoolAccess(void *base, uint32_t i)
{
    return (((char*)base)[i>>3] >> (i&0x7)) & 0x1;
}
inline void mmapBoolSet(void *base, uint32_t i, bool v)
{
    ((unsigned char*) base)[i>>3] =
        (((unsigned char*) base)[i>>3] & ~(1<<(i&0x7))) |
        ((v&1)<<(i&0x7));
}
inline size_t mmapBoolelementCount2Bytes(uint32_t i)
{
    return (size_t)(i+7)/8;
}

typedef MemoryMapArray<
bool,
uint32_t,
0xac6c1dc7,
20090109,
mmapBoolAccess,
mmapBoolSet,
mmapBoolelementCount2Bytes,
MemoryMapArrayHeader
> mmapArrayBool_t;

//
// define the two bit memory mapped array type:
//
inline uint32_t mmap2BitAccess(void *base, uint32_t i)
{
    return (((unsigned char*)base)[i>>2] >> ((i&0x3)<<1)) & 0x3;
}
inline void mmap2BitSet(void *base, uint32_t i, uint32_t v)
{
    ((unsigned char*) base)[i>>2] =
        (((unsigned char*) base)[i>>2]      // original value
         & ~(3<<((i&0x03)<<1)))              // logical AND off the original value
        | ((v&0x03)<<((i&0x3)<<1));         // logical OR in the new value
}
inline size_t mmap2BitElementCount2Bytes(uint32_t i)
{
    return (size_t)(i+3)/4;
}

typedef MemoryMapArray<
uint32_t,
uint32_t,
0x25b3ea5f,
20090109,
mmap2BitAccess,
mmap2BitSet,
mmap2BitElementCount2Bytes,
MemoryMapArrayHeader
> mmapArray2Bit_t;

//
// define the four bit memory mapped array type:
//
inline uint32_t mmap4BitAccess(void *base, uint32_t i)
{
    return (((unsigned char*)base)[i>>1] >> ((i&0x1)<<2)) & 0xf;
}
inline void mmap4BitSet(void *base, uint32_t i, uint32_t v)
{
    ((unsigned char*) base)[i>>1] =
        (((unsigned char*) base)[i>>1]      // original value
         & ~(7<<((i&0x01)<<2)))              // logical AND off the original value
        | ((v&0x0f)<<((i&0x1)<<2));         // logical OR in the new value
}
inline size_t mmap4BitelementCount2Bytes(uint32_t i)
{
    return (size_t)(i+1)/2;
}

typedef MemoryMapArray<
uint32_t,
uint32_t,
0x418e1874,
20090109,
mmap4BitAccess,
mmap4BitSet,
mmap4BitelementCount2Bytes,
MemoryMapArrayHeader
> mmapArray4Bit_t;

#if 0
// XXX this is example code I want to use to define arrays of genome wide match values
class   baseRecord
{
    unsigned int base:4;
    unsigned int qScore:7;
    unsigned int conflicts:5;   // how many cases of poorer matches that disagree
};

//
// define the baseRecord array type:
//
inline baseRecord &mmapBaseRecordAccess(void *base, uint32_t index)
{
    return *((baseRecord *)((char *)base + index*sizeof(baseRecord)));
}
inline void mmapBaseRecordSet(void *base, uint32_t index, baseRecord &v)
{
    mmapBaseRecordAccess(base, index) = v;
}
inline size_t mmapBaseRecordElementCount2Bytes(uint32_t i)
{
    return sizeof(baseRecord) * i;
}

typedef MemoryMapArray<
baseRecord &,
uint32_t,
0x12341234,
0xdeadbeef,
&mmapBaseRecordAccess,
mmapBaseRecordSet,
mmapBaseRecordElementCount2Bytes,
MemoryMapArrayHeader
> mmapArrayBaseRecord_t;
#endif

#endif