OpenGL Image Tiling Interpolation Problem

[Halcom]

Hi,

i have a frame grabber (at 30+ FPS) that give me images with sizes of 2560x2048 with RGB data in 8 Bit and 10 or 12 Bit data. A Single Texture is to large (really low performance) for older PC and a waste of memory. I wrote my own class derived from QGLWidget that shows me that image with zoom and so on. I am really happy that i t works because i never used openGL before. But then i interpolate i get some uggly artifacts at the tile borders.

I spend two days to find my bug, but i really dont know that is wrong and if you have some hints how i can optimize it i would be really happy.

The important methods are

newImageData - called from frame grabber thread and copies image to tiles
copyToTextureTile - copy image area to tile
paintGL - does the painting
drawTexture - draw the image tiles

Here my classes, cames in next post.

#include "GUIPixmapView.h"
 
#define TILESIZE 128
#define TILEBUFFERSIZE TILESIZE * TILESIZE * 4 * 2
 
class GUIGL3DView : public GUIOpenGLView
{
  Q_OBJECT
 
  public:
    GUIGL3DView(GUIGLPixmapView & iPixmapView, const QGLFormat & iFormat, QWidget * iParent = NULL);
 
    void newImageData();
 
  protected:
    void paintGL();
 
  private:
    struct TextureTile
    {
      unsigned char data[TILEBUFFERSIZE];
      size_t        offsetX;
      size_t        offsetY;
      size_t        width;
      size_t        height;
      size_t        tileX;
      size_t        tileY;
      GLuint        textureID;
 
      TextureTile()
      : offsetX(0)
      , offsetY(0)
      , width(0)
      , height(0)
      , tileX(-1)
      , tileY(-1)
      , textureID(0)
      {
        glGenTextures(1, &textureID);
      }
 
      ~TextureTile()
      {
        // delete texture
        if(textureID != 0)
          glDeleteTextures(1, &textureID);
      }
    };
 
    struct TextureData
    {
      QList<TextureTile*> tiles;
      size_t              usedTiles;
      size_t              bytesPerSample;
      size_t              samplesPerPixel;
      size_t              bitsPerSample;
 
      TextureData()
      : usedTiles(0)
      , bytesPerSample(0)
      , samplesPerPixel(0)
      , bitsPerSample(0)
      {}
 
      ~TextureData()
      {
        for(int i = 0; i < tiles.count(); i++)
          delete tiles.at(i);
      }
    };
 
    TextureData mImageTexture;
    GLint       mTileSize;
 
    void copyToTextureTile(TextureTile & oTile, int x, int y,  void * iSrcBuffer, size_t iWidth, size_t iHeight, size_t iBytesPerPixel);
 
    void drawLine();
 
    void drawTexture(TextureData & iTextureData);
};

#include "GUIPixmapView.h"

#define TILESIZE 128
#define TILEBUFFERSIZE TILESIZE * TILESIZE * 4 * 2

class GUIGL3DView : public GUIOpenGLView
{
  Q_OBJECT

  public:
    GUIGL3DView(GUIGLPixmapView & iPixmapView, const QGLFormat & iFormat, QWidget * iParent = NULL);

    void newImageData();

  protected:
    void paintGL();

  private:
    struct TextureTile
    {
      unsigned char data[TILEBUFFERSIZE];
      size_t        offsetX;
      size_t        offsetY;
      size_t        width;
      size_t        height;
      size_t        tileX;
      size_t        tileY;
      GLuint        textureID;

      TextureTile()
      : offsetX(0)
      , offsetY(0)
      , width(0)
      , height(0)
      , tileX(-1)
      , tileY(-1)
      , textureID(0)
      {
        glGenTextures(1, &textureID);
      }

      ~TextureTile()
      {
        // delete texture
        if(textureID != 0)
          glDeleteTextures(1, &textureID);
      }
    };

    struct TextureData
    {
      QList<TextureTile*> tiles;
      size_t              usedTiles;
      size_t              bytesPerSample;
      size_t              samplesPerPixel;
      size_t              bitsPerSample;

      TextureData()
      : usedTiles(0)
      , bytesPerSample(0)
      , samplesPerPixel(0)
      , bitsPerSample(0)
      {}

      ~TextureData()
      {
        for(int i = 0; i < tiles.count(); i++)
          delete tiles.at(i);
      }
    };

    TextureData mImageTexture;
    GLint       mTileSize;

    void copyToTextureTile(TextureTile & oTile, int x, int y,  void * iSrcBuffer, size_t iWidth, size_t iHeight, size_t iBytesPerPixel);

    void drawLine();

    void drawTexture(TextureData & iTextureData);
};

To copy to clipboard, switch view to plain text mode 

GUIGL3DView::GUIGL3DView(GUIGLPixmapView & iPixmapView, const QGLFormat & iFormat, QWidget * iParent)
: GUIOpenGLView(iPixmapView, iFormat, iParent)
, mImageTexture()
, mTileSize(0)
{
}
 
void GUIGL3DView::newImageData()
{
  if(mTileSize == 0)
  {
    mTileSize;
    glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &mTileSize);
    mTileSize = qMax(TILESIZE, mTileSize);
 
    if(mTileSize < TILESIZE)
      guiApp.showMessage(eInformation, tr("Warning your PC does not support textures with size of 2048 or more pixels. Textures size is set to %1 pixel. We can not ensure correct image representation.").arg(mTileSize));
  }
 
  // image values
  const unsigned int imageWidth = mPixmapView.pixmapWidth();
  const unsigned int imageHeight = mPixmapView.pixmapHeight();
 
  // get needed tile count
  const int xTiles = (imageWidth+(mTileSize-1))/mTileSize;
  const int yTiles = (imageHeight+(mTileSize-1))/mTileSize;
  const int tilesNeeded = yTiles * xTiles;
 
  // create missing tiles
  const int additionalTilesNeeded = tilesNeeded - mImageTexture.tiles.count();
  for(int i = 0; i < additionalTilesNeeded; i++)
  {
    TextureTile * newImageTile = new TextureTile();
    mImageTexture.tiles.append(newImageTile);
  }
 
  //remember number of currently used tiles
  mImageTexture.usedTiles = tilesNeeded;
 
  //create Texture
  if(mPixmapView.isExposureWarningEnabled() || !mPixmapView.pixmapIsNull())
  {
    // we only need to draw exposure or image, because exposure pixmap contains image data
    QImage image;
    if(mPixmapView.isExposureWarningEnabled())
      image = mPixmapView.mExposurePixmap.toImage();
    else
      image = mPixmapView.pixmap().toImage();
 
    const QImage::Format format = image.format();
    if(format == QImage::Format_Invalid)
      return;
 
    mImageTexture.bytesPerSample  = 1;
    mImageTexture.samplesPerPixel = 4;
    mImageTexture.bitsPerSample   = 8;
 
    // copy texture to tiles
    for(int x = 0; x < xTiles; x++)
      for(int y = 0; y < yTiles; y++)
        copyToTextureTile(*mImageTexture.tiles.at(y * xTiles + x), x, y, image.scanLine(0), image.width(), image.height(), mImageTexture.samplesPerPixel * mImageTexture.bytesPerSample);
  }
  else
  {
    ImageData data = mPixmapView.imageData();
    if(data.imageBuffer)
    {
      mImageTexture.bytesPerSample  = data.bytesPerSample;
      mImageTexture.samplesPerPixel = data.samplesPerPixel;
      mImageTexture.bitsPerSample   = data.bitsPerSample;
 
      // copy texture to tiles
      for(int x = 0; x < xTiles; x++)
        for(int y = 0; y < yTiles; y++)
          copyToTextureTile(*mImageTexture.tiles.at(y * xTiles + x), x, y, mPixmapView.imageData().imageBuffer, imageWidth, imageHeight, mImageTexture.samplesPerPixel * mImageTexture.bytesPerSample);
    }
  }
}
 
void GUIGL3DView::copyToTextureTile(TextureTile & oTile, int iX, int iY, void * iSrcBuffer, size_t iSrcWidth, size_t iSrcHeight, size_t iBytesPerPixel)
{
  oTile.tileX = iX;
  oTile.tileY = iY;
  oTile.offsetX = iX * mTileSize;
  oTile.offsetY = iY * mTileSize;
 
  const size_t endX = oTile.offsetX + mTileSize;
  const size_t endY = oTile.offsetY + mTileSize;
 
  if(endX > iSrcWidth)
    oTile.width =  iSrcWidth - oTile.offsetX;
  else
    oTile.width = mTileSize;
 
  if(endY > iSrcHeight)
    oTile.height =  iSrcHeight - oTile.offsetY;
  else
    oTile.height = mTileSize;
 
  const int srcBytesPerLine = iSrcWidth * iBytesPerPixel;
  const int dstBytesPerLine = mTileSize * iBytesPerPixel;
 
  const size_t srcFromX  = oTile.offsetX;
  const size_t srcFromY  = oTile.offsetY;
  const size_t srcWidth  = oTile.width;
  const size_t srcHeight = oTile.height;
 
  { // copy tile data
    unsigned char * source = (unsigned char*)iSrcBuffer + srcFromY * srcBytesPerLine + srcFromX * iBytesPerPixel;
    unsigned char * destination = (unsigned char*)oTile.data;
    const size_t bytesToCopy = srcWidth * iBytesPerPixel;
    for(size_t y = 0; y < srcHeight; y++)
    {
      memcpy(destination, source, bytesToCopy);
      source += srcBytesPerLine;
      destination += dstBytesPerLine;
    }
  }
}
 
void GUIGL3DView::paintGL()
{
  if(mPixmapView.isPaintBlocked())
    return;
 
  glViewport(0,0, width(), height());
 
  qglClearColor(mBackgroundColor);
  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_ACCUM_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 
  drawTexture(mImageTexture);
 
  checkErrors();
}
 
void GUIGL3DView::drawTexture(TextureData & iTextureData)
{
  if(iTextureData.usedTiles <= 0)
    return;
 
  // set projection - disable z-axis
  glMatrixMode(GL_PROJECTION);
 
  //clear projection matrix
  glLoadIdentity();
 
  //Creating an orthoscopic view matrix
  glOrtho(0, width(), height(), 0, -1, 1);
 
  //Define how alpha blending will work and enable alpha blending.
  glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
  glEnable(GL_BLEND);
 
  //Disabling the depth test (z will not be used to tell what object
  //will be shown above another, only the order in which I draw them.)
  glDisable(GL_DEPTH_TEST);
 
  // create image format data
  GLenum channels = GL_RGB;
  if(iTextureData.samplesPerPixel == 1)
    channels = GL_LUMINANCE;
  if(iTextureData.samplesPerPixel == 4)
    channels = GL_RGBA;
 
  GLenum dataType = GL_UNSIGNED_BYTE;
  if(iTextureData.bytesPerSample == 2)
    dataType = GL_UNSIGNED_SHORT;
 
  // get image data
  const float imageWidth = mPixmapView.pixmapWidth();
  const float imageHeight = mPixmapView.pixmapHeight();
 
  // center image
  const float viewLeft   = (width() - imageWidth * mZoom)/2;
  const float viewTop    = (height() - imageHeight * mZoom)/2;
 
  // move image to respect sliders
  const float offsetX = mPixmapView.mHScroll->isVisible() ? mPixmapView.mHScroll->value() : 0;
  const float offsetY = mPixmapView.mVScroll->isVisible() ? mPixmapView.mVScroll->value() : 0;
  glTranslatef(-offsetX * mZoom, -offsetY * mZoom, 0);
 
  // image start in upper left corner
  if(viewLeft < 0)
    glTranslatef(-viewLeft, 0, 0);
 
  if(viewTop < 0)
    glTranslatef(0, -viewTop, 0);
 
  // enable texturing
  glEnable(GL_TEXTURE_2D);
 
  // set scaling factor
  if(mImageTexture.bytesPerSample == 1)
    setScale(1.0);
  else
    setScale(1 << (16 - mImageTexture.bitsPerSample));
 
  // draw tiles
  const float tileSize = mTileSize * mZoom;
  for(size_t i = 0; i < iTextureData.usedTiles; i++)
  {
    TextureTile * tile = iTextureData.tiles.at(i);
 
    //specify texture to use
    glBindTexture(GL_TEXTURE_2D, tile->textureID);
 
    // set texturing parameters
    if(mPixmapView.isInterpolationEnabled() || mZoom <= 1.0)
    {
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    }
    else
    {
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    }
 
    // send texture to graphic card
    glTexImage2D(GL_TEXTURE_2D, 0, channels, mTileSize, mTileSize, 0, channels, dataType, tile->data);
 
    // get tile coordinates
    const float tileTop = viewTop + tile->tileY * tileSize;
    const float tileLeft = viewLeft + tile->tileX * tileSize;
 
    float tileBottom = viewTop + (tile->tileY + 1) * tileSize;
    if(tile->height != mTileSize)
      tileBottom = tileTop + tile->height * mZoom;
 
    float tileRight = viewLeft + (tile->tileX + 1) * tileSize;
    if(tile->width != mTileSize)
      tileRight = tileLeft + tile->width * mZoom;
 
    const float sourceBottom = (float)(tile->height) / (float) mTileSize;
    const float sourceRight = (float)(tile->width) / (float) mTileSize;
 
    glBegin(GL_QUADS);
      glTexCoord2f(sourceRight, 0);
      glVertex3f(tileRight , tileTop, 0);       //ro
 
      glTexCoord2f(0, 0);
      glVertex3f(tileLeft, tileTop, 0);         //lo
 
      glTexCoord2f(0, sourceBottom);
      glVertex3f(tileLeft, tileBottom, 0);      //lu
 
      glTexCoord2f(sourceRight, sourceBottom);
      glVertex3f(tileRight, tileBottom, 0);     //ru
    glEnd();
  }
}

GUIGL3DView::GUIGL3DView(GUIGLPixmapView & iPixmapView, const QGLFormat & iFormat, QWidget * iParent)
: GUIOpenGLView(iPixmapView, iFormat, iParent)
, mImageTexture()
, mTileSize(0)
{
}

void GUIGL3DView::newImageData()
{
  if(mTileSize == 0)
  {
    mTileSize;
    glGetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_WIDTH, &mTileSize);
    mTileSize = qMax(TILESIZE, mTileSize);

    if(mTileSize < TILESIZE)
      guiApp.showMessage(eInformation, tr("Warning your PC does not support textures with size of 2048 or more pixels. Textures size is set to %1 pixel. We can not ensure correct image representation.").arg(mTileSize));
  }

  // image values
  const unsigned int imageWidth = mPixmapView.pixmapWidth();
  const unsigned int imageHeight = mPixmapView.pixmapHeight();

  // get needed tile count
  const int xTiles = (imageWidth+(mTileSize-1))/mTileSize;
  const int yTiles = (imageHeight+(mTileSize-1))/mTileSize;
  const int tilesNeeded = yTiles * xTiles;

  // create missing tiles
  const int additionalTilesNeeded = tilesNeeded - mImageTexture.tiles.count();
  for(int i = 0; i < additionalTilesNeeded; i++)
  {
    TextureTile * newImageTile = new TextureTile();
    mImageTexture.tiles.append(newImageTile);
  }

  //remember number of currently used tiles
  mImageTexture.usedTiles = tilesNeeded;

  //create Texture
  if(mPixmapView.isExposureWarningEnabled() || !mPixmapView.pixmapIsNull())
  {
    // we only need to draw exposure or image, because exposure pixmap contains image data
    QImage image;
    if(mPixmapView.isExposureWarningEnabled())
      image = mPixmapView.mExposurePixmap.toImage();
    else
      image = mPixmapView.pixmap().toImage();

    const QImage::Format format = image.format();
    if(format == QImage::Format_Invalid)
      return;

    mImageTexture.bytesPerSample  = 1;
    mImageTexture.samplesPerPixel = 4;
    mImageTexture.bitsPerSample   = 8;

    // copy texture to tiles
    for(int x = 0; x < xTiles; x++)
      for(int y = 0; y < yTiles; y++)
        copyToTextureTile(*mImageTexture.tiles.at(y * xTiles + x), x, y, image.scanLine(0), image.width(), image.height(), mImageTexture.samplesPerPixel * mImageTexture.bytesPerSample);
  }
  else
  {
    ImageData data = mPixmapView.imageData();
    if(data.imageBuffer)
    {
      mImageTexture.bytesPerSample  = data.bytesPerSample;
      mImageTexture.samplesPerPixel = data.samplesPerPixel;
      mImageTexture.bitsPerSample   = data.bitsPerSample;

      // copy texture to tiles
      for(int x = 0; x < xTiles; x++)
        for(int y = 0; y < yTiles; y++)
          copyToTextureTile(*mImageTexture.tiles.at(y * xTiles + x), x, y, mPixmapView.imageData().imageBuffer, imageWidth, imageHeight, mImageTexture.samplesPerPixel * mImageTexture.bytesPerSample);
    }
  }
}

void GUIGL3DView::copyToTextureTile(TextureTile & oTile, int iX, int iY, void * iSrcBuffer, size_t iSrcWidth, size_t iSrcHeight, size_t iBytesPerPixel)
{
  oTile.tileX = iX;
  oTile.tileY = iY;
  oTile.offsetX = iX * mTileSize;
  oTile.offsetY = iY * mTileSize;

  const size_t endX = oTile.offsetX + mTileSize;
  const size_t endY = oTile.offsetY + mTileSize;

  if(endX > iSrcWidth)
    oTile.width =  iSrcWidth - oTile.offsetX;
  else
    oTile.width = mTileSize;

  if(endY > iSrcHeight)
    oTile.height =  iSrcHeight - oTile.offsetY;
  else
    oTile.height = mTileSize;

  const int srcBytesPerLine = iSrcWidth * iBytesPerPixel;
  const int dstBytesPerLine = mTileSize * iBytesPerPixel;

  const size_t srcFromX  = oTile.offsetX;
  const size_t srcFromY  = oTile.offsetY;
  const size_t srcWidth  = oTile.width;
  const size_t srcHeight = oTile.height;

  { // copy tile data
    unsigned char * source = (unsigned char*)iSrcBuffer + srcFromY * srcBytesPerLine + srcFromX * iBytesPerPixel;
    unsigned char * destination = (unsigned char*)oTile.data;
    const size_t bytesToCopy = srcWidth * iBytesPerPixel;
    for(size_t y = 0; y < srcHeight; y++)
    {
      memcpy(destination, source, bytesToCopy);
      source += srcBytesPerLine;
      destination += dstBytesPerLine;
    }
  }
}

void GUIGL3DView::paintGL()
{
  if(mPixmapView.isPaintBlocked())
    return;

  glViewport(0,0, width(), height());

  qglClearColor(mBackgroundColor);
  glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_ACCUM_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);

  drawTexture(mImageTexture);

  checkErrors();
}

void GUIGL3DView::drawTexture(TextureData & iTextureData)
{
  if(iTextureData.usedTiles <= 0)
    return;

  // set projection - disable z-axis
  glMatrixMode(GL_PROJECTION);

  //clear projection matrix
  glLoadIdentity();

  //Creating an orthoscopic view matrix
  glOrtho(0, width(), height(), 0, -1, 1);

  //Define how alpha blending will work and enable alpha blending.
  glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
  glEnable(GL_BLEND);

  //Disabling the depth test (z will not be used to tell what object
  //will be shown above another, only the order in which I draw them.)
  glDisable(GL_DEPTH_TEST);

  // create image format data
  GLenum channels = GL_RGB;
  if(iTextureData.samplesPerPixel == 1)
    channels = GL_LUMINANCE;
  if(iTextureData.samplesPerPixel == 4)
    channels = GL_RGBA;

  GLenum dataType = GL_UNSIGNED_BYTE;
  if(iTextureData.bytesPerSample == 2)
    dataType = GL_UNSIGNED_SHORT;

  // get image data
  const float imageWidth = mPixmapView.pixmapWidth();
  const float imageHeight = mPixmapView.pixmapHeight();

  // center image
  const float viewLeft   = (width() - imageWidth * mZoom)/2;
  const float viewTop    = (height() - imageHeight * mZoom)/2;

  // move image to respect sliders
  const float offsetX = mPixmapView.mHScroll->isVisible() ? mPixmapView.mHScroll->value() : 0;
  const float offsetY = mPixmapView.mVScroll->isVisible() ? mPixmapView.mVScroll->value() : 0;
  glTranslatef(-offsetX * mZoom, -offsetY * mZoom, 0);

  // image start in upper left corner
  if(viewLeft < 0)
    glTranslatef(-viewLeft, 0, 0);

  if(viewTop < 0)
    glTranslatef(0, -viewTop, 0);

  // enable texturing
  glEnable(GL_TEXTURE_2D);

  // set scaling factor
  if(mImageTexture.bytesPerSample == 1)
    setScale(1.0);
  else
    setScale(1 << (16 - mImageTexture.bitsPerSample));

  // draw tiles
  const float tileSize = mTileSize * mZoom;
  for(size_t i = 0; i < iTextureData.usedTiles; i++)
  {
    TextureTile * tile = iTextureData.tiles.at(i);

    //specify texture to use
    glBindTexture(GL_TEXTURE_2D, tile->textureID);

    // set texturing parameters
    if(mPixmapView.isInterpolationEnabled() || mZoom <= 1.0)
    {
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    }
    else
    {
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
      glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
    }

    // send texture to graphic card
    glTexImage2D(GL_TEXTURE_2D, 0, channels, mTileSize, mTileSize, 0, channels, dataType, tile->data);

    // get tile coordinates
    const float tileTop = viewTop + tile->tileY * tileSize;
    const float tileLeft = viewLeft + tile->tileX * tileSize;

    float tileBottom = viewTop + (tile->tileY + 1) * tileSize;
    if(tile->height != mTileSize)
      tileBottom = tileTop + tile->height * mZoom;

    float tileRight = viewLeft + (tile->tileX + 1) * tileSize;
    if(tile->width != mTileSize)
      tileRight = tileLeft + tile->width * mZoom;

    const float sourceBottom = (float)(tile->height) / (float) mTileSize;
    const float sourceRight = (float)(tile->width) / (float) mTileSize;

    glBegin(GL_QUADS);
      glTexCoord2f(sourceRight, 0);
      glVertex3f(tileRight , tileTop, 0);       //ro

      glTexCoord2f(0, 0);
      glVertex3f(tileLeft, tileTop, 0);         //lo

      glTexCoord2f(0, sourceBottom);
      glVertex3f(tileLeft, tileBottom, 0);      //lu

      glTexCoord2f(sourceRight, sourceBottom);
      glVertex3f(tileRight, tileBottom, 0);     //ru
    glEnd();
  }
}

To copy to clipboard, switch view to plain text mode 

[Halcom]

Found the Problem!!! Have to give the texture a Border line so openGl can interpolate to and give him correct texture coordinates.