extern_bridge.hpp

Go to the documentation of this file.

#ifndef FSTEXTERNBRIDGE_H
#define FSTEXTERNBRIDGE_H
 /*
=========================================================================
Copyright and license:
  * FST4 software (with the exception of any bundled or externally linked
    third-party components) is Copyright (C) 2000-present Institute of
    Information Theory and Automation of the Czech Academy of Sciences (UTIA).
  * FST4 is free software: you can redistribute it and/or modify it under
    the terms of the GNU Affero General Public License as published by the
    Free Software Foundation, version 3. The full text of the license is in
    the file LICENSE in the distribution root and at
    https://www.gnu.org/licenses/agpl-3.0.html.
  * FST4 is distributed in the hope that it will be useful, but WITHOUT ANY
    WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
    License for more details.
  * Closed-source use without the AGPL's copyleft obligations requires a
    commercial license from UTIA; in case of interest write to
    utia@utia.cas.cz. LICENSING.md in the distribution root explains both
    options, the contributor license agreement that every contribution
    requires, and the contacts (technical questions and pull requests:
    somol@utia.cas.cz, with "FST" in the subject line).
========================================================================== */
 
#include <string>
#include <vector>
#include <cstdio>
#include <cstdlib>
#include <cassert>
#include <iostream>
#include <sstream>
#include <mutex>
#include "error.hpp"
#include "global.hpp"
 
// ---- the ONLY platform-sensitive process plumbing in FST4, kept in this single header
//      so that both external-tool bridges share one implementation instead of duplicating
//      the fork/CreateProcess code ----
#ifdef _WIN32
        #ifndef NOMINMAX
                #define NOMINMAX // keep windows.h from defining min/max macros that break std::min/max and numeric_limits textually after this include
        #endif
        #ifndef WIN32_LEAN_AND_MEAN
                #define WIN32_LEAN_AND_MEAN
        #endif
        #include <windows.h>
        #include <io.h>
        #include <fcntl.h>
#else
        #include <unistd.h>
        #include <sys/wait.h>
        #include <signal.h>
        #include <fcntl.h>
#endif
 
namespace FST {

inline std::mutex& extern_spawn_mutex() {static std::mutex m; return m;}

class Extern_Bridge {
public:
        explicit Extern_Bridge(const char *component) : _component(component) {init_process_handles();} 
        ~Extern_Bridge() {shutdown_process();}
        Extern_Bridge(const Extern_Bridge&) = delete; // owners construct fresh bridges for their clones instead
        Extern_Bridge& operator=(const Extern_Bridge&) = delete;
 
        void set_command(const std::string &cmd) {assert(!running()); _cmd=cmd;} 
        const std::string& get_command() const {return _cmd;} 
        bool running() const {return _to_child!=0;} 
        std::FILE* to_child() {assert(_to_child); return _to_child;} 
        void flush_to_child() {assert(_to_child); std::fflush(_to_child);} 
        void die_on_write() const {throw fst_error(_component+": the external command '"+_cmd+"' stopped accepting input (died?)");} 

        void ensure_process()
        {
                if(running()) return;
                if(_cmd.empty()) throw fst_error(_component+": no command set (see --extern-cmd)");
                // serialize spawns process-wide: without this, a concurrently spawning clone's child
                // (threaded search = one child per criterion clone) can inherit this clone's pipe ends
                // in the window before they are marked non-inheritable below - an inherited write end
                // keeps a sibling child's stdin open past shutdown_process()'s fclose, the child never
                // sees EOF, and the destructor's waitpid deadlocks
                std::unique_lock<std::mutex> spawn_lock(extern_spawn_mutex());
                { // one-time runtime notice: someone running an untrusted third-party config
                  // should see that it executes code; the CLI's prose plumbing swallows this under
                  // `--verbosity` silent and routes it to stderr under machine output formats
                        static bool notice_fired=false; // guarded by the spawn lock
                        if(!notice_fired) {
                                notice_fired=true;
                                std::ostringstream sos; sos << "NOTE: --extern-cmd executes user-supplied code through the system shell: " << _cmd << std::endl;
                                syncout::print(std::cout,sos);
                        }
                }
#ifdef _WIN32
                // ---- Windows plumbing (CreateProcess through cmd.exe) ----
                SECURITY_ATTRIBUTES sa; sa.nLength=sizeof(sa); sa.bInheritHandle=TRUE; sa.lpSecurityDescriptor=NULL;
                HANDLE in_rd=0,in_wr=0,out_rd=0,out_wr=0;
                if(!CreatePipe(&in_rd,&in_wr,&sa,0) || !CreatePipe(&out_rd,&out_wr,&sa,0))
                        throw fst_error(_component+": CreatePipe failed");
                SetHandleInformation(in_wr,HANDLE_FLAG_INHERIT,0);  // our ends must not leak to the child
                SetHandleInformation(out_rd,HANDLE_FLAG_INHERIT,0);
                STARTUPINFOA si; ZeroMemory(&si,sizeof(si)); si.cb=sizeof(si);
                si.dwFlags=STARTF_USESTDHANDLES;
                si.hStdInput=in_rd; si.hStdOutput=out_wr; si.hStdError=GetStdHandle(STD_ERROR_HANDLE);
                PROCESS_INFORMATION pi; ZeroMemory(&pi,sizeof(pi));
                std::string full=std::string("cmd.exe /C ")+_cmd;
                std::vector<char> cmdbuf(full.begin(),full.end()); cmdbuf.push_back('\0');
                if(!CreateProcessA(NULL,cmdbuf.data(),NULL,NULL,TRUE,0,NULL,NULL,&si,&pi))
                        {CloseHandle(in_rd);CloseHandle(in_wr);CloseHandle(out_rd);CloseHandle(out_wr);
                         throw fst_error(_component+": cannot launch '"+_cmd+"'");}
                CloseHandle(pi.hThread);
                CloseHandle(in_rd); CloseHandle(out_wr); // child's ends
                _child_process=pi.hProcess;
                _to_child=_fdopen(_open_osfhandle((intptr_t)in_wr,_O_WRONLY|_O_BINARY),"wb");
                _from_child=_fdopen(_open_osfhandle((intptr_t)out_rd,_O_RDONLY|_O_BINARY),"rb");
                if(!_to_child || !_from_child) {shutdown_process(); throw fst_error(_component+": cannot attach pipes to '"+_cmd+"'");}
#else
                // ---- POSIX plumbing (fork + /bin/sh -c) ----
                int to_pipe[2], from_pipe[2];
                if(pipe(to_pipe)!=0 || pipe(from_pipe)!=0) throw fst_error(_component+": pipe() failed");
                fcntl(to_pipe[1],F_SETFD,FD_CLOEXEC);   // our ends must not leak into later children
                fcntl(from_pipe[0],F_SETFD,FD_CLOEXEC); // (the Windows twin: SetHandleInformation above)
                // SIGPIPE discipline: a child that stops reading (crashed mid-search) must surface as
                // the send-path fst_errors, not kill fst4 silently (default SIGPIPE disposition
                // terminates the process before fputs/fputc can even return EOF - observed as exit 141
                // with zero output, intermittently masked by kernel pipe buffering on small payloads)
#ifdef F_SETNOSIGPIPE
                fcntl(to_pipe[1],F_SETNOSIGPIPE,1); // macOS/BSD: per-fd, write fails with EPIPE instead of raising SIGPIPE
#else
                // no per-fd control on this platform, so SIGPIPE is ignored PROCESS-WIDE and stays
                // ignored for the rest of the run - a side effect on the whole embedding program,
                // documented as a warning on ensure_process() above
                signal(SIGPIPE,SIG_IGN);
#endif
                const pid_t pid=fork();
                if(pid<0) throw fst_error(_component+": fork() failed");
                if(pid==0) { // child: wire the pipes to stdin/stdout and exec the command via the shell
                        dup2(to_pipe[0],0); dup2(from_pipe[1],1);
                        close(to_pipe[0]); close(to_pipe[1]); close(from_pipe[0]); close(from_pipe[1]);
                        execl("/bin/sh","sh","-c",_cmd.c_str(),(char*)0);
                        _exit(127); // exec failed
                }
                close(to_pipe[0]); close(from_pipe[1]);
                _child_pid=pid;
                _to_child=fdopen(to_pipe[1],"w");
                _from_child=fdopen(from_pipe[0],"r");
                if(!_to_child || !_from_child) {shutdown_process(); throw fst_error(_component+": cannot attach pipes to '"+_cmd+"'");}
#endif
        }

        void shutdown_process()
        {
                if(_to_child) {std::fclose(_to_child); _to_child=0;} // EOF on the child's stdin = terminate request
                if(_from_child) {std::fclose(_from_child); _from_child=0;}
#ifdef _WIN32
                if(_child_process) {WaitForSingleObject(_child_process,2000); TerminateProcess(_child_process,0); CloseHandle(_child_process); _child_process=0;}
#else
                if(_child_pid>0) {int status=0; waitpid(_child_pid,&status,0); _child_pid=-1;}
#endif
        }

        void send_line(const std::string &line)
        {
                assert(_to_child);
                if(std::fputs(line.c_str(),_to_child)==EOF || std::fputc('\n',_to_child)==EOF)
                        die_on_write();
        }

        std::string read_line()
        {
                assert(_from_child);
                std::string out;
                char buf[256];
                for(;;)
                {
                        if(!std::fgets(buf,sizeof(buf),_from_child))
                                throw fst_error(_component+": the external command '"+_cmd+"' produced no response (died?)");
                        out+=buf;
                        if(!out.empty() && out.back()=='\n') {out.pop_back(); if(!out.empty() && out.back()=='\r') out.pop_back(); return out;}
                        if(out.size()>1048576) // a child streaming bytes without newline must fail loudly, not exhaust memory
                                {shutdown_process(); throw fst_error(_component+": the external command '"+_cmd+"' produced a response line exceeding 1 MB (runaway output?)");}
                }
        }
 
private:
        std::string _component; 
        std::string _cmd;       
        std::FILE *_to_child;   
        std::FILE *_from_child; 
#ifdef _WIN32
        HANDLE _child_process;
#else
        pid_t _child_pid;       
#endif
        void init_process_handles()
        {
                _to_child=0; _from_child=0;
#ifdef _WIN32
                _child_process=0;
#else
                _child_pid=-1;
#endif
        }
};
 
} // namespace
#endif // FSTEXTERNBRIDGE_H ///:~