Using a named (fifo) pipe to transport arrays (images) between python and c++
<p>I need to send an array (representing an image) through a named FIFO pipe from a python process to a c++ process, and then back the other way (on a Linux system). </p>

<p>The below code works great when using named pipes between two Python processes. It uses numpy's tostring() and fromstring() functions:</p>

<p><strong>Send frames over named pipe (Python)</strong>
</p>

<pre><code>import cv2
import numpy as np
from time import sleep

##########################################################

FIFO_Images = "./../pipes/images.fifo"
videoName = "./../../videos/videoName.avi"
delim = "break"

##########################################################


def sendImage(h, w, d, pixelarray):
    imageString = pixelarray.tostring()
    with open(FIFO_Images, "w") as f:
        f.write(str(h)+ delim + str(w)+ delim + str(d) + delim + imageString)
    sleep(.01)  
    return 

##########################################################

cap = cv2.VideoCapture(videoName)

while(cap.isOpened()):
    ret, frame_rgb = cap.read() 
    h, w, d = frame_rgb.shape
    sendImage(h, w, d, frame_rgb)

cap.release()
cv2.destroyAllWindows()
</code></pre>

<p><strong>Read frames over named pipe (Python)</strong>
</p>

<pre><code>import cv2
import numpy as np

##########################################################

FIFO_Images = "./../pipes/images.fifo"
delim = "break"

##########################################################

def getFrame():
    with open(FIFO_Images, "r") as f:
        data = f.read().split(delim)

        #parse incoming string, which has format (height, width, depth, imageData)        
        h=int(data[0])
        w=int(data[1])
        d=int(data[2])
        imageString = data[3]

        #convert array string into numpy array
        array = np.fromstring(imageString, dtype=np.uint8)

        #reshape numpy array into the required dimensions
        frame = array.reshape((h,w,d))

        return frame  

##########################################################

while(True):

    frame = getFrame()

    cv2.imshow('frame', frame)
    cv2.waitKey(1) &amp; 0xFF 
</code></pre>

<p>However, I couldn't figure out how to read the entire image from the pipe on the cpp side, since it takes "\n" as a delimiter for the read automatically. </p>

<p>My workaround was to do a base64 encoding on the "tostring()" image, then send <em>that</em> over the pipe. This works, but the base64 decoding on the other slide is much too slow for real-time applications (~0.2 seconds per frame). Code: </p>

<p><strong>Send base64-encoded images over named pipe (Python)</strong>
</p>

<pre><code>import cv2
import numpy as np
from time import time
from time import sleep
import base64

##########################################################

FIFO_Images = "./../pipes/images.fifo"
videoName = "./../../videos/videoName.avi"

delim = ";;"

##########################################################

def sendImage(h, w, d, pixelarray):


    flat = pixelarray.flatten()

    imageString = base64.b64encode(pixelarray.tostring())
    fullString = str(h)+ delim + str(w)+ delim + str(d)+ delim + imageString + delim + "\n"


    with open(FIFO_Images, "w") as f:
        f.write(fullString)

    return 

##########################################################

cap = cv2.VideoCapture(videoName)
count = 0

while(cap.isOpened()):
    ret, frame_rgb = cap.read()
    h, w, d = frame_rgb.shape

    frame_gbr = cv2.cvtColor(frame_rgb, cv2.COLOR_RGB2BGR)

    sendImage(h, w, d, frame_rgb)


cap.release()
cv2.destroyAllWindows()
</code></pre>

<p><strong>Read base64-encoded images over named pipe (C++)</strong>
</p>

<pre><code>#include "opencv2/opencv.hpp"

#include &lt;iostream&gt;
#include &lt;fstream&gt;
#include &lt;string&gt;
#include &lt;vector&gt;
#include &lt;unistd.h&gt;
#include &lt;fcntl.h&gt;

#include &lt;stdio.h&gt;
#include &lt;stdlib.h&gt;
#include &lt;sys/stat.h&gt;
#include &lt;linux/stat.h&gt;
#include &lt;ctime&gt;

using namespace std;
using namespace cv;

#define FIFO_FILE "./../../../pipes/images.fifo"
#define MAX_BUF 10000000

FILE *fp;
char readbuf[MAX_BUF + 1];  //add 1 to the expected size to accomodate the mysterious "extra byte", which I think signals the end of the line. 


/************************BASE64 Decoding*********************************************/
std::string base64_encode(unsigned char const* , unsigned int len);
std::string base64_decode(std::string const&amp; s);

static const std::string base64_chars = 
             "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
             "abcdefghijklmnopqrstuvwxyz"
             "0123456789+/";


static inline bool is_base64(unsigned char c) {
  return (isalnum(c) || (c == '+') || (c == '/'));
}

std::string base64_encode(unsigned char const* bytes_to_encode, unsigned int in_len) {
  std::string ret;
  int i = 0;
  int j = 0;
  unsigned char char_array_3[3];
  unsigned char char_array_4[4];

  while (in_len--) {
    char_array_3[i++] = *(bytes_to_encode++);
    if (i == 3) {
      char_array_4[0] = (char_array_3[0] &amp; 0xfc) &gt;&gt; 2;
      char_array_4[1] = ((char_array_3[0] &amp; 0x03) &lt;&lt; 4) + ((char_array_3[1] &amp; 0xf0) &gt;&gt; 4);
      char_array_4[2] = ((char_array_3[1] &amp; 0x0f) &lt;&lt; 2) + ((char_array_3[2] &amp; 0xc0) &gt;&gt; 6);
      char_array_4[3] = char_array_3[2] &amp; 0x3f;

      for(i = 0; (i &lt;4) ; i++)
        ret += base64_chars[char_array_4[i]];
      i = 0;
    }
  }

  if (i)
  {
    for(j = i; j &lt; 3; j++)
      char_array_3[j] = '\0';

    char_array_4[0] = (char_array_3[0] &amp; 0xfc) &gt;&gt; 2;
    char_array_4[1] = ((char_array_3[0] &amp; 0x03) &lt;&lt; 4) + ((char_array_3[1] &amp; 0xf0) &gt;&gt; 4);
    char_array_4[2] = ((char_array_3[1] &amp; 0x0f) &lt;&lt; 2) + ((char_array_3[2] &amp; 0xc0) &gt;&gt; 6);
    char_array_4[3] = char_array_3[2] &amp; 0x3f;

    for (j = 0; (j &lt; i + 1); j++)
      ret += base64_chars[char_array_4[j]];

    while((i++ &lt; 3))
      ret += '=';

  }

  return ret;

}

std::string base64_decode(std::string const&amp; encoded_string) {
  int in_len = encoded_string.size();
  int i = 0;
  int j = 0;
  int in_ = 0;
  unsigned char char_array_4[4], char_array_3[3];
  std::string ret;

  while (in_len-- &amp;&amp; ( encoded_string[in_] != '=') &amp;&amp; is_base64(encoded_string[in_])) {
    char_array_4[i++] = encoded_string[in_]; in_++;
    if (i ==4) {
      for (i = 0; i &lt;4; i++)
        char_array_4[i] = base64_chars.find(char_array_4[i]);

      char_array_3[0] = (char_array_4[0] &lt;&lt; 2) + ((char_array_4[1] &amp; 0x30) &gt;&gt; 4);
      char_array_3[1] = ((char_array_4[1] &amp; 0xf) &lt;&lt; 4) + ((char_array_4[2] &amp; 0x3c) &gt;&gt; 2);
      char_array_3[2] = ((char_array_4[2] &amp; 0x3) &lt;&lt; 6) + char_array_4[3];

      for (i = 0; (i &lt; 3); i++)
        ret += char_array_3[i];
      i = 0;
    }
  }

  if (i) {
    for (j = i; j &lt;4; j++)
      char_array_4[j] = 0;

    for (j = 0; j &lt;4; j++)
      char_array_4[j] = base64_chars.find(char_array_4[j]);

    char_array_3[0] = (char_array_4[0] &lt;&lt; 2) + ((char_array_4[1] &amp; 0x30) &gt;&gt; 4);
    char_array_3[1] = ((char_array_4[1] &amp; 0xf) &lt;&lt; 4) + ((char_array_4[2] &amp; 0x3c) &gt;&gt; 2);
    char_array_3[2] = ((char_array_4[2] &amp; 0x3) &lt;&lt; 6) + char_array_4[3];

    for (j = 0; (j &lt; i - 1); j++) ret += char_array_3[j];
  }

  return ret;
}

/*********************************************************************/

int stringToInt(string str)
{
    int num;
    if (!(istringstream(str) &gt;&gt; num)) num = 0;

    return num;
}

/*********************************************************************/
bool timerOn = 0;
clock_t timerStart;

void Timer(string process)
{

    if (!timerOn)
    {
        timerStart = clock();
        timerOn = true;
    }

    else if (timerOn)
    {
        double duration = (clock() - timerStart) / (double) CLOCKS_PER_SEC; 
        cout &lt;&lt; "Time to complete: ";
        printf("%.2f", duration);
        cout &lt;&lt; ": " &lt;&lt; process &lt;&lt; endl;
        timerOn = false;
    }
}

/*********************************************************************/

void getFrame()
{
    string fullString;
    string delimiter = ";;";
    size_t pos = 0;
    string token;

    int h;
    int w;
    int d;
    string imgString;

    int fifo;
    bool cont(true);

    /***************************
    Read from the pipe
    www.tldp.org/LDP/lpg/node18.html
    ***************************/
    Timer("Read from pipe");
    fp = fopen(FIFO_FILE, "r");
    fgets(readbuf, MAX_BUF + 1, fp); // Stops when MAX_BUF characters are read, the newline character ("\n") is read, or the EOF (end of file) is reached
    string line(readbuf);
    fclose(fp);
    Timer("Read from pipe");

    //////parse the string into components

    Timer("Parse string");
    int counter = 0;
    while ((pos = line.find(delimiter)) != string::npos)
    {
        token = line.substr(0,pos);

        if (counter == 0)
        {
            h = stringToInt(token);
        }
        else if (counter == 1)
        {
            w = stringToInt(token);
        }
        else if (counter == 2)
        {
            d = stringToInt(token);
        }
        else if (counter == 3)
        {
            imgString = token;
            //cout &lt;&lt; imgString[0] &lt;&lt; endl;
        }
        else
        {
            cout &lt;&lt; "ERROR: Too many paramaters passed" &lt;&lt; endl;
            return;
        }

        line.erase(0, pos + delimiter.length());

        counter ++;
    }

    if (counter == 3)
    {
        imgString = token;
    }

    if (counter &lt; 3)
    {
        cout &lt;&lt; "ERROR: Not enough paramaters passed: " &lt;&lt; counter &lt;&lt; endl;
        //return;
    }

    Timer("Parse string");



    /***************************
    Convert from Base64
    ***************************/      
    Timer("Decode Base64");
    std::string decoded = base64_decode(imgString);
    Timer("Decode Base64");

    /***************************
    Convert to vector of ints
    ***************************/
    Timer("Convert to vector of ints");   
    std::vector&lt;uchar&gt; imgVector;   
    for (int i = 0; i &lt; decoded.length(); i = i+1) // + 4)
    {
        int temp =  (char(decoded[i]));
        imgVector.push_back(temp);
    }
    Timer("Convert to vector of ints");

    //////convert the vector into a matrix
    Mat frame = Mat(imgVector).reshape(d, h);   

    namedWindow("Frame", WINDOW_AUTOSIZE);
    imshow("Frame", frame);

    waitKey(1); 
}


int main()
{

    /* Create the FIFO if it does not exist */
    umask(0);
    mknod(FIFO_FILE, S_IFIFO|0666, 0);

    while(1)
    {
        getFrame();

    }

    return 0;
}
</code></pre>

<p>There must be a more efficient way to accomplish this. Can anyone make a recommendation? While I'm happy to hear suggestions for other methods to accomplish this, <strong>I am constrained to using named pipes</strong> for now. </p>

<p>This is overcomplicated. If you need to send binary data, send their length first, then newline (<code>\n</code>), and then the data (raw, no base64). Receive it on the other side by readling a line, parsing the number and then just reading a block of data of given length.</p>

<p><strong>Example</strong> - writing binary data to a FIFO (or file) in Python:</p>

<pre><code>#!/usr/bin/env python3

import os

fifo_name = 'fifo'

def main():
    data = b'blob\n\x00 123'
    try:
        os.mkfifo(fifo_name)
    except FileExistsError:
        pass
    with open(fifo_name, 'wb') as f:
        # b for binary mode
        f.write('{}\n'.format(len(data)).encode())
        f.write(data)

if __name__ == '__main__':
    main()
</code></pre>

<p>Reading binary data from FIFO in C++:</p>

<pre><code>#include &lt;iostream&gt;
#include &lt;fstream&gt;
#include &lt;string&gt;
#include &lt;vector&gt;

#include &lt;sys/stat.h&gt;

int main(int argc, char *argv[]) {
    const char *fifo_name = "fifo";
    mknod(fifo_name, S_IFIFO | 0666, 0);
    std::ifstream f(fifo_name);
    std::string line;
    getline(f, line);
    auto data_size = std::stoi(line);
    std::cout &lt;&lt; "Size: " &lt;&lt; data_size &lt;&lt; std::endl;
    std::string data;
    {
        std::vector&lt;char&gt; buf(data_size);
        f.read(buf.data(), data_size);
        // write to vector data is valid since C++11
        data.assign(buf.data(), buf.size());
    }
    if (!f.good()) {
        std::cerr &lt;&lt; "Read failed" &lt;&lt; std::endl;
    }
    std::cout &lt;&lt; "Data size: " &lt;&lt; data.size() &lt;&lt; " content: " &lt;&lt; data &lt;&lt; std::endl;
}
</code></pre>

python c++ linux pipe ipc