Add h264 encoder support

common/rfb/CMakeLists.txt

@@ -71,9 +71,9 @@ target_link_libraries(rfb ${JPEG_LIBRARIES} ${PIXMAN_LIBRARIES})
 target_link_directories(rfb PUBLIC ${PIXMAN_LIBRARY_DIRS})
 
 if(ENABLE_H264 AND NOT H264_LIBS STREQUAL "NONE")
-  target_sources(rfb PRIVATE H264Decoder.cxx H264DecoderContext.cxx)
+  target_sources(rfb PRIVATE H264Decoder.cxx H264DecoderContext.cxx H264Encoder.cxx H264EncoderContext.cxx)
   if(H264_LIBS STREQUAL "LIBAV")
-    target_sources(rfb PRIVATE H264LibavDecoderContext.cxx)
+    target_sources(rfb PRIVATE H264LibavDecoderContext.cxx H264LibavEncoderContext.cxx)
   elseif(H264_LIBS STREQUAL "WIN")
     target_sources(rfb PRIVATE H264WinDecoderContext.cxx)
   endif()

common/rfb/EncodeManager.cxx

@@ -40,6 +40,7 @@
 #include <rfb/ZRLEEncoder.h>
 #include <rfb/TightEncoder.h>
 #include <rfb/TightJPEGEncoder.h>
+#include <rfb/H264Encoder.h>
 
 using namespace rfb;
 
@@ -68,6 +69,7 @@ enum EncoderClass {
   encoderTight,
   encoderTightJPEG,
   encoderZRLE,
+  encoderH264,
   encoderClassMax,
 };
 
@@ -103,6 +105,8 @@ static const char *encoderClassName(EncoderClass klass)
     return "Tight (JPEG)";
   case encoderZRLE:
     return "ZRLE";
+  case encoderH264:
+    return "H264";
   case encoderClassMax:
     break;
   }
@@ -146,6 +150,7 @@ EncodeManager::EncodeManager(SConnection* conn_)
   encoders[encoderTight] = new TightEncoder(conn);
   encoders[encoderTightJPEG] = new TightJPEGEncoder(conn);
   encoders[encoderZRLE] = new ZRLEEncoder(conn);
+  encoders[encoderH264] = new H264Encoder(conn);
 
   updates = 0;
   memset(&copyStats, 0, sizeof(copyStats));
@@ -246,6 +251,9 @@ bool EncodeManager::supported(int encoding)
   case encodingHextile:
   case encodingZRLE:
   case encodingTight:
+#ifdef HAVE_H264
+  case encodingH264:
+#endif
     return true;
   default:
     return false;
@@ -406,6 +414,14 @@ void EncodeManager::prepareEncoders(bool allowLossy)
     bitmapRLE = indexedRLE = encoderZRLE;
     bitmap = indexed = encoderZRLE;
     break;
+#ifdef HAVE_H264
+  case encodingH264:
+    vlog.debug("prepareEncoders --- encodingH264");
+    fullColour = encoderH264;
+    bitmapRLE = indexedRLE = encoderH264;
+    bitmap = indexed = encoderH264;
+    break;
+#endif
   }
 
   // Any encoders still unassigned?

common/rfb/H264Encoder.cxx

@@ -0,0 +1,54 @@
+#include <rdr/OutStream.h>
+#include <rfb/encodings.h>
+#include <rfb/Exception.h>
+#include <rfb/SConnection.h>
+#include <rfb/PixelBuffer.h>
+#include <rfb/LogWriter.h>
+#include <rfb/H264EncoderContext.h>
+#include <rfb/H264Encoder.h>
+
+using namespace rfb;
+
+static LogWriter vlog("H264Encoder");
+
+H264Encoder::H264Encoder(SConnection* conn_)
+  : Encoder(conn_, encodingH264, EncoderPlain)
+{
+}
+
+H264Encoder::~H264Encoder()
+{
+}
+
+bool H264Encoder::isSupported()
+{
+  vlog.debug("isSupported");
+  return conn->client.supportsEncoding(encodingH264);
+}
+
+void H264Encoder::writeRect(const PixelBuffer* pb, const Palette& /*palette*/)
+{
+  H264EncoderContext* ctx =
+    H264EncoderContext::createContext(pb->width(), pb->height());
+  if (!ctx) {
+    throw std::runtime_error("H264Encoder: Context not be created");
+  }
+  if (!ctx->isReady()) {
+    throw std::runtime_error("H264Encoder: Context is not ready");
+  }
+
+  rdr::OutStream* os;
+  os = conn->getOutStream();
+  vlog.debug("writeRect width(%d), height(%d), bpp(%d)", pb->width(),
+             pb->height(), pb->getPF().bpp);
+  ctx->encode(os, pb);
+  ctx->reset();
+}
+
+void H264Encoder::writeSolidRect(int width, int height, const PixelFormat& pf,
+                                 const uint8_t* colour)
+{
+  (void)colour;
+  vlog.info("H264Encoder writeSolidRect, width %d, height %d, pf.bpp %d",
+            width, height, pf.bpp);
+}

common/rfb/H264Encoder.h

@@ -0,0 +1,17 @@
+#ifndef __RFB_H264ENCODER_H__
+#define __RFB_H264ENCODER_H__
+
+#include <rfb/Encoder.h>
+
+namespace rfb {
+  class H264Encoder : public Encoder {
+  public:
+    H264Encoder(SConnection* conn);
+    virtual ~H264Encoder();
+    bool isSupported() override;
+    void writeRect(const PixelBuffer* pb, const Palette& palette) override;
+    void writeSolidRect(int width, int height, const PixelFormat& pf,
+                        const uint8_t* colour) override;
+  };
+}
+#endif

common/rfb/H264EncoderContext.cxx

@@ -0,0 +1,32 @@
+#include <os/Mutex.h>
+#include <rfb/LogWriter.h>
+#include <rfb/H264EncoderContext.h>
+#include <rfb/H264LibavEncoderContext.h>
+
+using namespace rfb;
+
+static LogWriter vlog("H264EncoderContext");
+
+H264EncoderContext* H264EncoderContext::createContext(int width, int height)
+{
+  H264EncoderContext* ret = new H264LibavEncoderContext();
+  if (!ret->initCodec(width, height)) {
+    throw std::runtime_error("H264EncoderContext: Unable to create context");
+  }
+  return ret;
+}
+
+H264EncoderContext::~H264EncoderContext()
+{
+}
+
+bool H264EncoderContext::isReady()
+{
+  os::AutoMutex lock(&mutex);
+  return initialized;
+}
+
+void H264EncoderContext::reset()
+{
+  freeCodec();
+}

common/rfb/H264EncoderContext.h

@@ -0,0 +1,34 @@
+#ifndef __RFB_H264ENCODERCONTEXT_H__
+#define __RFB_H264ENCODERCONTEXT_H__
+
+#include <stdint.h>
+
+#include <os/Mutex.h>
+#include <rfb/Rect.h>
+#include <rfb/Encoder.h>
+#include <rdr/OutStream.h>
+
+namespace rfb {
+  class H264EncoderContext {
+  public:
+    static H264EncoderContext* createContext(int width, int height);
+    virtual ~H264EncoderContext() = 0;
+    virtual void encode(rdr::OutStream* os, const PixelBuffer* pb) = 0;
+    void reset();
+    bool isReady();
+
+  protected:
+    os::Mutex mutex;
+    bool initialized;
+
+    H264EncoderContext()
+    {
+      initialized = false;
+    }
+
+    virtual bool initCodec(int width, int height) = 0;
+    virtual void freeCodec() = 0;
+  };
+}
+
+#endif