From b020910f786fd7033c7e905961886072e6533755 Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Wed, 14 Jan 2026 01:00:34 -0500 Subject: [PATCH 01/17] added support for reading stdout and stderr separately from processes --- src/async-process.c | 402 +++++++++++++++++++++++++++++++++----------- src/async-process.h | 37 +++- src/test.c | 37 ++++ 3 files changed, 367 insertions(+), 109 deletions(-) create mode 100644 src/test.c diff --git a/src/async-process.c b/src/async-process.c index ad58157..ec73da7 100644 --- a/src/async-process.c +++ b/src/async-process.c @@ -1,133 +1,331 @@ #include "async-process.h" -static const char* open_pty(int *out_fd) -{ - int fd = posix_openpt(O_RDWR | O_CLOEXEC | O_NOCTTY); - if (fd < 0) return NULL; - if (grantpt(fd) == -1 || unlockpt(fd) == -1) return NULL; - fcntl(fd, F_SETFD, FD_CLOEXEC); - const char *name = ptsname(fd); - if (name == NULL) { - close(fd); - return NULL; - } - *out_fd = fd; - return name; +int init_str(struct str *str) { + str->buf = malloc(sizeof(char) * 256); + if (str->buf == NULL) + return -1; + + str->len = 0; + str->cap = 256; + + return 0; } -static struct process* allocate_process(int fd, const char *pts_name, int pid) -{ - struct process *process = malloc(sizeof(struct process)); - if (process == NULL) +void del_str(struct str *str) { + free(str->buf); +} + +static struct process* allocate_process(int fd_io, + int fd_er, + const char *pts_io_name, + const char *pts_er_name, + int pid) { + int stdout_ret = 0, stderr_ret = 0; + char *io_str = NULL, *er_str = NULL; + + struct process *process = malloc(sizeof(struct process)); + if (process == NULL) + return NULL; + + stdout_ret = init_str(&process->stdout); + if (stdout_ret == -1) + goto FAILED_MALLOC; + + stderr_ret = init_str(&process->stderr); + if (stderr_ret == -1) + goto FAILED_MALLOC; + + io_str = malloc(strlen(pts_io_name) + 1); + if (io_str == NULL) + goto FAILED_MALLOC; + + er_str = malloc(strlen(pts_er_name) + 1); + if (er_str == NULL) + goto FAILED_MALLOC; + + strcpy(io_str, pts_io_name); + strcpy(er_str, pts_er_name); + + process->pts_io_name = io_str; + process->pts_er_name = er_str; + process->fd_io = fd_io; + process->fd_er = fd_er; + process->pid = pid; + + return process; + +FAILED_MALLOC: + if (process != NULL) free(process); + if (stdout_ret == -1) del_str(&process->stdout); + if (stderr_ret == -1) del_str(&process->stderr); + if (io_str != NULL) free(io_str); + if (er_str != NULL) free(er_str); return NULL; - process->fd = fd; - process->pty_name = malloc(strlen(pts_name) + 1); - process->pid = pid; - strcpy(process->pty_name, pts_name); - return process; +} + +void delete_process(struct process *process) { + kill(process->pid, 9); + close(process->fd_io); + close(process->fd_er); + free(process->stdout.buf); + free(process->stderr.buf); + free(process->pts_io_name); + free(process->pts_er_name); + free(process); } void my_exit(int status) { - // exitを使うとatexitで動作に影響を与えられる、これが原因でプロセスを終了できなくなる事があるので使うのを避ける - // 例えばSDL2はat_exitを使っているせいか、lemのSDL2 frontendでasync_processが動作しなくなっていた - _exit(status); + // exitを使うとatexitで動作に影響を与えられる、これが原因でプロセスを終了できなくなる事があるので使うのを避ける + // 例えばSDL2はat_exitを使っているせいか、lemのSDL2 frontendでasync_processが動作しなくなっていた + _exit(status); } -struct process* create_process(char *const command[], bool nonblock, const char *path) -{ - int pty_master; - const char *pts_name = open_pty(&pty_master); - if (pts_name == NULL) - return NULL; +// opens a PTY and assigns master and slave file descriptors to fdm and fds +// respectively. Name will be malloced and it is the callers responsibility +// to free name. On success, return 0. On fail, returns -1. All references +// will also be either initialized or set -1/NULL appropriately. +int open_pty(int *fdm, int *fds, char **name) { + *fdm = -1; + *fds = -1; + *name = NULL; + + // gets a PTY, and initializes the attached slave PTS. grantpt and unlockpt + // are required before opening the slave device. + *fdm = posix_openpt(O_RDWR | O_NOCTTY); + if (*fdm == -1 || grantpt(*fdm) == -1 || unlockpt(*fdm) == -1) + goto FAILED_SETUP; + + // ptsname returns a string that must be copied, as it is overwritten + // on subsequent calls. + const char *tmp = ptsname(*fdm); + if (tmp == NULL) + goto FAILED_SETUP; + + size_t tmp_len = strlen(tmp); + *name = malloc(tmp_len * sizeof(char)); + if (*name == NULL) + goto FAILED_SETUP; + + memcpy(*name, tmp, tmp_len+1); + + *fds = open(*name, O_RDWR | O_NOCTTY); + if (*fds == -1) + goto FAILED_SETUP; + + // ensure both slave and master close after program finishes + fcntl(*fdm, F_SETFD, FD_CLOEXEC); + fcntl(*fds, F_SETFD, FD_CLOEXEC); + + // set master as non-blocking (for get_process_output functions) + fcntl(*fdm, F_SETFL, O_NONBLOCK); + + // Set raw mode + struct termios tty; + tcgetattr(*fds, &tty); + cfmakeraw(&tty); + tcsetattr(*fds, TCSANOW, &tty); + + return 0; + +FAILED_SETUP: + if (*fdm != -1) close(*fdm); + if (*fds != -1) close(*fds); + if (*name != NULL) free(*name); + + *fdm = -1; + *fds = -1; + *name = NULL; + return -1; +} + +struct process* create_process(char *const command[], const char *path) { + // Unix PTYs are bi-directional communication streams. Typically, a terminal will + // combine stdout and stderr and display them in the same output. We want to + // keep the outputs separate at this level so master_pty_er is created just to carry + // the stderr stream. + // + // There is a potential bug here, if the process tries to set terminal attributes (like + // with stty), these updates won't be propogated across both terminals. + + int master_pty_io, slave_pts_io, master_pty_er, slave_pts_er; + char *pts_io_name, *pts_er_name; + int ret; + + ret = open_pty(&master_pty_io, &slave_pts_io, &pts_io_name); + if (ret == -1) + goto FAILED_SETUP; + + ret = open_pty(&master_pty_er, &slave_pts_er, &pts_er_name); + if (ret == -1) + goto FAILED_SETUP; + + // START CHILD PROCESS AND RETURN ITS PID + pid_t pid = fork(); + + if (pid == -1) { + goto FAILED_SETUP; + } else if (pid != 0) { + close(slave_pts_io); + close(slave_pts_er); + // parent process, return process structure. + struct process *p = allocate_process(master_pty_io, master_pty_er, + pts_io_name, pts_er_name, pid); + + // allocate_process copies the strings it is passed, open_pty mallocs strings + // so we need to free them here before we exit. + free(pts_io_name); + free(pts_er_name); + return p; + } + + // VVV CHILD PROCESS VVV + setsid(); + + // we don't need these in the child process. + free(pts_io_name); + free(pts_er_name); + close(master_pty_io); + close(master_pty_er); - if (nonblock) - fcntl(pty_master, F_SETFL, O_NONBLOCK); - - int pipefd[2]; - - if (pipe(pipefd) == -1) return NULL; - - pid_t pid = fork(); - - if (pid == 0) { - close(pipefd[0]); - pid = fork(); - if (pid == 0) { - close(pipefd[1]); - setsid(); - int pty_slave = open(pts_name, O_RDWR | O_NOCTTY); - close(pty_master); - - // Set raw mode - struct termios tty; - tcgetattr(pty_slave, &tty); - cfmakeraw(&tty); - tcsetattr(pty_slave, TCSANOW, &tty); - - dup2(pty_slave, STDIN_FILENO); - dup2(pty_slave, STDOUT_FILENO); - dup2(pty_slave, STDERR_FILENO); - close(pty_slave); - if (path != NULL) chdir(path); - execvp(command[0], command); - int error_status = errno; - if (error_status == ENOENT) { + dup2(slave_pts_io, STDIN_FILENO); + dup2(slave_pts_io, STDOUT_FILENO); + dup2(slave_pts_er, STDERR_FILENO); + + close(slave_pts_io); + close(slave_pts_er); + + if (path != NULL) chdir(path); + + // run command, the current fork process will switch to + // the command. + execvp(command[0], command); + + // if execution reaches here, there was a problem starting + // the program. execvp does not return on success. + int error_status = errno; + if (error_status == ENOENT) { char str[128]; sprintf(str, "%s: command not found", command[0]); write(STDIN_FILENO, str, strlen(str)); - } else { + } else { char *str = strerror(error_status); write(STDIN_FILENO, str, strlen(str)); - } - my_exit(error_status); - } else { - char buf[12]; - sprintf(buf, "%d", pid); - write(pipefd[1], buf, strlen(buf)+1); - close(pipefd[1]); - my_exit(0); } - } else { - close(pipefd[1]); - if (waitpid(pid, NULL, 0) == -1) - return NULL; - char buf[12]; - read(pipefd[0], buf, sizeof(buf)); - close(pipefd[0]); - return allocate_process(pty_master, pts_name, atoi(buf)); - } + my_exit(error_status); - return NULL; + // ERROR HANDLING +FAILED_SETUP: + // we can assume at this point that any FD that is not -1 needs closed. + if (master_pty_io != -1) close(master_pty_io); + if (master_pty_er != -1) close(master_pty_er); + if (slave_pts_io != -1) close(slave_pts_io); + if (slave_pts_er != -1) close(slave_pts_er); + + // we can assume that any name pointer that is not NULL needs free. + if (pts_io_name != NULL) free(pts_io_name); + if (pts_er_name != NULL) free(pts_er_name); + + return NULL; } -void delete_process(struct process *process) -{ - kill(process->pid, 9); - close(process->fd); - free(process->pty_name); - free(process); +int process_pid(struct process *process) { + return process->pid; } -int process_pid(struct process *process) -{ - return process->pid; +void process_write_string(struct process *process, const char *string) { + write(process->fd_io, string, strlen(string)); } -void process_send_input(struct process *process, const char *string) -{ - write(process->fd, string, strlen(string)); +void process_write(struct process *process, const char *buf, size_t n) { + write(process->fd_io, buf, n); } -const char* process_receive_output(struct process *process) -{ - int n = read(process->fd, process->buffer, sizeof(process->buffer)-1); - if (n == -1) - return NULL; - process->buffer[n] = '\0'; - return process->buffer; +// reads all data available in fd (should be non-blocking) into str, +// returns number of bytes read on success, -1 on error. +int str_read_fd(struct str *str, int fd) { + int total_read = 0; + while (true) { + // resize buffer if it is too small + if (str->cap - str->len <= 1) { + char *new_ptr = realloc(str->buf, 2*str->cap); + if (new_ptr == NULL) + return -1; + str->buf = new_ptr; + str->cap *= 2; + } + + // read as much data from fd as possible. + int n = read(fd, str->buf + str->len, str->cap - str->len); + + if (total_read == 0 && n == -1) { + return -1; // an error occured on first read + } else if (n <= 0) { + return total_read; + } + + total_read += n; + str->len += n; + } + + return -1; // control flow shouldn't reach here. +} + +char* _process_receive_fd(struct str *s, int fd) { + int n = str_read_fd(s, fd); + if (n == -1) + return NULL; + + char *ret = malloc((s->len + 1) * sizeof(char)); + if (ret == NULL) + return NULL; + + memcpy(ret, s->buf, s->len); + ret[s->len] = '\0'; + + s->len = 0; + return ret; +} + +char* process_receive_stdout(struct process *p) { + return _process_receive_fd(&p->stdout, p->fd_io); +} + +char* process_receive_stderr(struct process *p) { + return _process_receive_fd(&p->stderr, p->fd_er); +} + +char* process_receive_output(struct process *process) { + char *stdout = process_receive_stdout(process); + char *stderr = process_receive_stderr(process); + + if (stdout == NULL && stderr == NULL) + return NULL; + + if (stdout != NULL && stderr == NULL) + return stdout; + + if (stdout == NULL && stderr != NULL) + return stderr; + + size_t o_len = strlen(stdout); + size_t e_len = strlen(stderr); + size_t length = o_len + e_len + 1; + char *ret = malloc(length * sizeof(char)); + if (ret == NULL) + return NULL; + + memcpy(ret, stdout, o_len); + memcpy(ret + o_len, stderr, e_len); + + ret[length] = '\0'; + + free(stdout); + free(stderr); + + return ret; } int process_alive_p(struct process *process) { - return kill(process->pid, 0) == 0; + return kill(process->pid, 0) == 0; } diff --git a/src/async-process.h b/src/async-process.h index dd2b26f..ed2f8f0 100644 --- a/src/async-process.h +++ b/src/async-process.h @@ -18,18 +18,41 @@ #include #include +struct str { + char* buf; + size_t len; + size_t cap; +}; + +int init_str(struct str *str); +void del_str(struct str *str); +int str_read_fd(struct str *str, int fd); + struct process { - char buffer[1024*4]; - int fd; - char *pty_name; - pid_t pid; + struct str stdout; + struct str stderr; + + int fd_io; + int fd_er; + char *pts_io_name; + char *pts_er_name; + pid_t pid; }; -struct process* create_process(char *const command[], bool nonblock, const char *path); +struct process* create_process(char *const command[], const char *path); void delete_process(struct process *process); int process_pid(struct process *process); -void process_send_input(struct process *process, const char *string); -const char* process_receive_output(struct process *process); + +void process_write(struct process *process, const char* buf, size_t n); +void process_write_string(struct process *process, const char *string); + +/** receive process stdout. MUST FREE RETURNED PONTER */ +char* process_receive_stdout(struct process *process); +/** receive process stderr. MUST FREE RETURNED PONTER */ +char* process_receive_stderr(struct process *process); +/** receive process stdout and stderr (one after another). +MUST FREE RETURNED PONTER */ +char* process_receive_output(struct process *process); int process_alive_p(struct process *process); #endif diff --git a/src/test.c b/src/test.c new file mode 100644 index 0000000..2dc7fb6 --- /dev/null +++ b/src/test.c @@ -0,0 +1,37 @@ +#include "async-process.h" + +int main() { + char *cmd[] = {"clangd"}; + struct process *p = create_process(cmd, NULL); + + struct str s; + init_str(&s); + + while (true) { + int n = str_read_fd(&s, STDIN_FILENO); + if (n > 0) { + process_write(p, s.buf, n); + s.len = 0; + } + + + char *out = NULL; + char *err = NULL; + + out = process_receive_stdout(p); + err = process_receive_stderr(p); + + if (out != NULL) { + printf("%s", out); + free(out); + } + + if (err != NULL) { + printf("\033[31m%s\033[0m", err); + free(err); + } + } + + delete_process(p); + return 0; +} From a479cac5ed095a7c108bbca87384c6cc7f1251a4 Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Wed, 14 Jan 2026 02:40:26 -0500 Subject: [PATCH 02/17] updated cffi bindings to match new C lib API --- src/async-process.lisp | 49 +++++++++++++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/src/async-process.lisp b/src/async-process.lisp index a88b3c6..3d104e9 100644 --- a/src/async-process.lisp +++ b/src/async-process.lisp @@ -5,7 +5,8 @@ :process-send-input :process-receive-output :process-alive-p - :create-process)) + :create-process + :cffi-test)) (in-package :async-process) (eval-when (:compile-toplevel :load-toplevel :execute) @@ -48,7 +49,7 @@ (:unix "libasyncprocess.so") (:windows "libasyncprocess.dll")) -(cffi:use-foreign-library async-process) +(cffi:use-foreign-library #P"/home/ethan/Documents/async-process/.libs/libasyncprocess.so") (defclass process () ((process :reader process-process :initarg :process) @@ -56,7 +57,6 @@ (cffi:defcfun ("create_process" %create-process) :pointer (command :pointer) - (nonblock :boolean) (path :string)) (cffi:defcfun ("delete_process" %delete-process) :void @@ -65,17 +65,30 @@ (cffi:defcfun ("process_pid" %process-pid) :int (process :pointer)) -(cffi:defcfun ("process_send_input" %process-send-input) :void +(cffi:defcfun ("process_write" %process-write) :void + (process :pointer) + (string :string) + (n :size)) + +(cffi:defcfun ("process_write_string" %process-write-string) :void (process :pointer) (string :string)) +(cffi:defcfun ("process_receive_stdout" %process-receive-stdout) :pointer + (process :pointer)) + +(cffi:defcfun ("process_receive_stderr" %process-receive-stderr) :pointer + (process :pointer)) + (cffi:defcfun ("process_receive_output" %process-receive-output) :pointer (process :pointer)) (cffi:defcfun ("process_alive_p" %process-alive-p) :boolean (process :pointer)) -(defun create-process (command &key nonblock (encode cffi:*default-foreign-encoding*) directory) +(cffi:defcfun "cffi_test" :string) + +(defun create-process (command &key (encode cffi:*default-foreign-encoding*) directory) (when (and directory (not (uiop:directory-exists-p directory))) (error "Directory ~S does not exist" directory)) (let* ((command (uiop:ensure-list command)) @@ -85,9 +98,9 @@ :for c :in command :do (setf (cffi:mem-aref argv :string i) c)) (setf (cffi:mem-aref argv :string length) (cffi:null-pointer)) - (let ((p (%create-process argv nonblock (if directory - (namestring directory) - (cffi:null-pointer))))) + (let ((p (%create-process argv (if directory + (namestring directory) + (cffi:null-pointer))))) (if (cffi:null-pointer-p p) (error "create-process failed: ~S" command) (make-instance 'process :process p :encode encode)))))) @@ -100,7 +113,7 @@ (defun process-send-input (process string) (let ((cffi:*default-foreign-encoding* (process-encode process))) - (%process-send-input (process-process process) string))) + (%process-write-string (process-process process) string))) (defun pointer-to-string (pointer) (unless (cffi:null-pointer-p pointer) @@ -116,9 +129,21 @@ ;; Fallback when an error occurs with UTF-8 encoding (map 'string #'code-char octets)))))) -(defun process-receive-output (process) - (let ((cffi:*default-foreign-encoding* (process-encode process))) - (pointer-to-string (%process-receive-output (process-process process))))) +(defun process-receive-output (process &optional (source :both)) + "`source` can be either `:stdout`, `:stderr`, or `:both`. It specifies the stream +to read from." + (flet ((call-cfun (read-func) + "helper function to call one of the three cffi functions for receiving output." + (let ((cffi:*default-foreign-encoding* (process-encode process)) + (output (funcall read-func (process-process process)))) + (prog1 + (pointer-to-string output) + (cffi:foreign-free output))))) + + (case source + (:stdout (call-cfun '%process-receive-stdout)) + (:stderr (call-cfun '%process-receive-stderr)) + (:both (call-cfun '%process-receive-output))))) (defun process-alive-p (process) (%process-alive-p (process-process process))) From 3cc06ef51de2fff8f5f193996fc4b26667ff645f Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Fri, 16 Jan 2026 00:44:21 -0500 Subject: [PATCH 03/17] fixed memory leaks/errors identified with valgrind --- src/async-process.c | 21 ++++++++++++--------- src/test.c | 8 +++++--- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/async-process.c b/src/async-process.c index ec73da7..846fae2 100644 --- a/src/async-process.c +++ b/src/async-process.c @@ -22,6 +22,8 @@ static struct process* allocate_process(int fd_io, int pid) { int stdout_ret = 0, stderr_ret = 0; char *io_str = NULL, *er_str = NULL; + size_t io_strlen = strlen(pts_io_name) + 1; + size_t er_strlen = strlen(pts_er_name) + 1; struct process *process = malloc(sizeof(struct process)); if (process == NULL) @@ -35,16 +37,16 @@ static struct process* allocate_process(int fd_io, if (stderr_ret == -1) goto FAILED_MALLOC; - io_str = malloc(strlen(pts_io_name) + 1); + io_str = malloc(io_strlen * sizeof(char)); if (io_str == NULL) goto FAILED_MALLOC; - er_str = malloc(strlen(pts_er_name) + 1); + er_str = malloc(er_strlen * sizeof(char)); if (er_str == NULL) goto FAILED_MALLOC; - strcpy(io_str, pts_io_name); - strcpy(er_str, pts_er_name); + memcpy(io_str, pts_io_name, io_strlen); + memcpy(er_str, pts_er_name, er_strlen); process->pts_io_name = io_str; process->pts_er_name = er_str; @@ -67,8 +69,8 @@ void delete_process(struct process *process) { kill(process->pid, 9); close(process->fd_io); close(process->fd_er); - free(process->stdout.buf); - free(process->stderr.buf); + del_str(&process->stdout); + del_str(&process->stderr); free(process->pts_io_name); free(process->pts_er_name); free(process); @@ -101,12 +103,12 @@ int open_pty(int *fdm, int *fds, char **name) { if (tmp == NULL) goto FAILED_SETUP; - size_t tmp_len = strlen(tmp); + size_t tmp_len = strlen(tmp) + 1; *name = malloc(tmp_len * sizeof(char)); if (*name == NULL) goto FAILED_SETUP; - memcpy(*name, tmp, tmp_len+1); + memcpy(*name, tmp, tmp_len); *fds = open(*name, O_RDWR | O_NOCTTY); if (*fds == -1) @@ -245,7 +247,7 @@ void process_write(struct process *process, const char *buf, size_t n) { int str_read_fd(struct str *str, int fd) { int total_read = 0; while (true) { - // resize buffer if it is too small + // resize buffer if it is too small (include space for null terminator) if (str->cap - str->len <= 1) { char *new_ptr = realloc(str->buf, 2*str->cap); if (new_ptr == NULL) @@ -260,6 +262,7 @@ int str_read_fd(struct str *str, int fd) { if (total_read == 0 && n == -1) { return -1; // an error occured on first read } else if (n <= 0) { + str->buf[str->len] = '\0'; return total_read; } diff --git a/src/test.c b/src/test.c index 2dc7fb6..783a310 100644 --- a/src/test.c +++ b/src/test.c @@ -1,7 +1,7 @@ #include "async-process.h" int main() { - char *cmd[] = {"clangd"}; + char *cmd[] = {"tee", "ima-cool-file", NULL}; struct process *p = create_process(cmd, NULL); struct str s; @@ -12,8 +12,9 @@ int main() { if (n > 0) { process_write(p, s.buf, n); s.len = 0; + if (strcmp(s.buf, "exit\n") == 0) + break; } - char *out = NULL; char *err = NULL; @@ -29,9 +30,10 @@ int main() { if (err != NULL) { printf("\033[31m%s\033[0m", err); free(err); - } + } } delete_process(p); + del_str(&s); return 0; } From 69aad356a2e3a77474d53685a68b384c95bb7b66 Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Fri, 16 Jan 2026 01:25:02 -0500 Subject: [PATCH 04/17] process_write handles large amounts of data better, returns bytes written --- src/async-process.c | 21 +++++++++++++++++---- src/async-process.h | 4 ++-- src/test.c | 23 ++++++++++++++++++++++- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/async-process.c b/src/async-process.c index 846fae2..0ce25c1 100644 --- a/src/async-process.c +++ b/src/async-process.c @@ -234,12 +234,25 @@ int process_pid(struct process *process) { return process->pid; } -void process_write_string(struct process *process, const char *string) { - write(process->fd_io, string, strlen(string)); +ssize_t process_write_string(struct process *process, const char *string) { + return process_write(process, string, strlen(string)); } -void process_write(struct process *process, const char *buf, size_t n) { - write(process->fd_io, buf, n); +ssize_t process_write(struct process *process, const char *buf, size_t n) { + ssize_t bytes_written = 0; + + while (bytes_written < n) { + ssize_t sent = write(process->fd_io, buf + bytes_written, n - bytes_written); + if (bytes_written != 0 && sent == -1) { + return bytes_written; + } else if (sent == -1) { + return -1; + } + + bytes_written += sent; + } + + return bytes_written; } // reads all data available in fd (should be non-blocking) into str, diff --git a/src/async-process.h b/src/async-process.h index ed2f8f0..a68273a 100644 --- a/src/async-process.h +++ b/src/async-process.h @@ -43,8 +43,8 @@ struct process* create_process(char *const command[], const char *path); void delete_process(struct process *process); int process_pid(struct process *process); -void process_write(struct process *process, const char* buf, size_t n); -void process_write_string(struct process *process, const char *string); +ssize_t process_write(struct process *process, const char* buf, size_t n); +ssize_t process_write_string(struct process *process, const char *string); /** receive process stdout. MUST FREE RETURNED PONTER */ char* process_receive_stdout(struct process *process); diff --git a/src/test.c b/src/test.c index 783a310..765babd 100644 --- a/src/test.c +++ b/src/test.c @@ -7,6 +7,27 @@ int main() { struct str s; init_str(&s); + #define TEST_INPUT_SIZE 50000 + char test_input[TEST_INPUT_SIZE]; + + for (size_t i = 0; i < TEST_INPUT_SIZE; i++) { + test_input[i] = '0' + (i % 10); + } + + size_t n = 0; + while (n != TEST_INPUT_SIZE) { + ssize_t bytes = process_write(p, test_input+n, TEST_INPUT_SIZE-n); + if (bytes > 0) { + printf("writing %d/%d bytes...\n", bytes, n); + n += bytes; + } else { + printf("error: %s\n", strerror(errno)); + } + + } + + printf("I just attempted to write %d.\nI wrote %d bytes.\n", TEST_INPUT_SIZE, n); + while (true) { int n = str_read_fd(&s, STDIN_FILENO); if (n > 0) { @@ -15,7 +36,7 @@ int main() { if (strcmp(s.buf, "exit\n") == 0) break; } - + char *out = NULL; char *err = NULL; From bb48843777da29e75a2028cfee472f30e9fb11ee Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Fri, 16 Jan 2026 20:47:01 -0500 Subject: [PATCH 05/17] process_write can now send arbitrarily large amounts of data --- src/async-process.c | 140 +++++++++++++++++++++++++------------------- src/async-process.h | 59 ++++++++++++++++--- src/test.c | 18 +++--- 3 files changed, 141 insertions(+), 76 deletions(-) diff --git a/src/async-process.c b/src/async-process.c index 0ce25c1..abadc17 100644 --- a/src/async-process.c +++ b/src/async-process.c @@ -20,7 +20,7 @@ static struct process* allocate_process(int fd_io, const char *pts_io_name, const char *pts_er_name, int pid) { - int stdout_ret = 0, stderr_ret = 0; + int stdout_ret = -1, stderr_ret = -1, both_ret = -1; char *io_str = NULL, *er_str = NULL; size_t io_strlen = strlen(pts_io_name) + 1; size_t er_strlen = strlen(pts_er_name) + 1; @@ -36,6 +36,10 @@ static struct process* allocate_process(int fd_io, stderr_ret = init_str(&process->stderr); if (stderr_ret == -1) goto FAILED_MALLOC; + + both_ret = init_str(&process->both); + if (both_ret == -1) + goto FAILED_MALLOC; io_str = malloc(io_strlen * sizeof(char)); if (io_str == NULL) @@ -58,8 +62,9 @@ static struct process* allocate_process(int fd_io, FAILED_MALLOC: if (process != NULL) free(process); - if (stdout_ret == -1) del_str(&process->stdout); - if (stderr_ret == -1) del_str(&process->stderr); + if (stdout_ret != -1) del_str(&process->stdout); + if (stderr_ret != -1) del_str(&process->stderr); + if (both_ret != -1) del_str(&process->both); if (io_str != NULL) free(io_str); if (er_str != NULL) free(er_str); return NULL; @@ -71,6 +76,7 @@ void delete_process(struct process *process) { close(process->fd_er); del_str(&process->stdout); del_str(&process->stderr); + del_str(&process->both); free(process->pts_io_name); free(process->pts_er_name); free(process); @@ -234,27 +240,6 @@ int process_pid(struct process *process) { return process->pid; } -ssize_t process_write_string(struct process *process, const char *string) { - return process_write(process, string, strlen(string)); -} - -ssize_t process_write(struct process *process, const char *buf, size_t n) { - ssize_t bytes_written = 0; - - while (bytes_written < n) { - ssize_t sent = write(process->fd_io, buf + bytes_written, n - bytes_written); - if (bytes_written != 0 && sent == -1) { - return bytes_written; - } else if (sent == -1) { - return -1; - } - - bytes_written += sent; - } - - return bytes_written; -} - // reads all data available in fd (should be non-blocking) into str, // returns number of bytes read on success, -1 on error. int str_read_fd(struct str *str, int fd) { @@ -286,61 +271,98 @@ int str_read_fd(struct str *str, int fd) { return -1; // control flow shouldn't reach here. } -char* _process_receive_fd(struct str *s, int fd) { +const char* _process_receive_fd(struct str *s, int fd) { int n = str_read_fd(s, fd); if (n == -1) return NULL; - char *ret = malloc((s->len + 1) * sizeof(char)); - if (ret == NULL) - return NULL; - - memcpy(ret, s->buf, s->len); - ret[s->len] = '\0'; - - s->len = 0; - return ret; + return s->buf; } -char* process_receive_stdout(struct process *p) { - return _process_receive_fd(&p->stdout, p->fd_io); +const char* process_receive_stdout(struct process *p) { + const char *r = _process_receive_fd(&p->stdout, p->fd_io); + p->stdout.len = 0; + return r; } -char* process_receive_stderr(struct process *p) { - return _process_receive_fd(&p->stderr, p->fd_er); +const char* process_receive_stderr(struct process *p) { + const char *r = _process_receive_fd(&p->stderr, p->fd_er); + p->stderr.len = 0; + return r; } -char* process_receive_output(struct process *process) { - char *stdout = process_receive_stdout(process); - char *stderr = process_receive_stderr(process); +const char* process_receive_output(struct process *p) { + _process_receive_fd(&p->stdout, p->fd_io); + _process_receive_fd(&p->stderr, p->fd_er); - if (stdout == NULL && stderr == NULL) - return NULL; + // these lengths include null terminators + size_t stdout_len = p->stdout.len; + size_t stderr_len = p->stderr.len; + + if (p->both.cap < (stdout_len + stderr_len)) { + char *new_ptr = realloc(p->both.buf, stdout_len + stderr_len); + if (new_ptr == NULL) + return NULL; + p->both.buf = new_ptr; + p->both.cap = stdout_len + stderr_len; + } + + // don't copy null terminator + if (stdout_len > 0 && p->stdout.buf[stdout_len] == '\0') { + stdout_len--; + } - if (stdout != NULL && stderr == NULL) - return stdout; + memcpy(p->both.buf, p->stdout.buf, stdout_len); + memcpy(p->both.buf + stdout_len, p->stderr.buf, stderr_len); // copy null terminator this time. - if (stdout == NULL && stderr != NULL) - return stderr; + p->both.len = 0; + p->stdout.len = 0; + p->stderr.len = 0; - size_t o_len = strlen(stdout); - size_t e_len = strlen(stderr); - size_t length = o_len + e_len + 1; - char *ret = malloc(length * sizeof(char)); - if (ret == NULL) - return NULL; + return p->both.buf; +} - memcpy(ret, stdout, o_len); - memcpy(ret + o_len, stderr, e_len); +ssize_t _process_write(struct process *process, const char *buf, size_t n, bool readp) { + ssize_t bytes_written = 0; - ret[length] = '\0'; + while (bytes_written < n) { + ssize_t sent = write(process->fd_io, buf + bytes_written, n - bytes_written); + + if (readp) { + _process_receive_fd(&process->stdout, process->fd_io); + _process_receive_fd(&process->stderr, process->fd_er); + } - free(stdout); - free(stderr); + if (bytes_written != 0 && sent == -1) { + return bytes_written; + } else if (sent == -1) { + return -1; + } + + bytes_written += sent; + } + + return bytes_written; +} + +ssize_t process_write(struct process *process, const char *buf, size_t n) { + return _process_write(process, buf, n, true); +} - return ret; +ssize_t process_write_string(struct process *process, const char *string) { + return _process_write(process, string, strlen(string), true); +} + +ssize_t process_write_noread(struct process *process, const char *buf, size_t n) { + return _process_write(process, buf, n, false); } +ssize_t process_write_string_noread(struct process *process, const char *string) { + return _process_write(process, string, strlen(string), false); +} + + + int process_alive_p(struct process *process) { return kill(process->pid, 0) == 0; diff --git a/src/async-process.h b/src/async-process.h index a68273a..d5467c0 100644 --- a/src/async-process.h +++ b/src/async-process.h @@ -31,6 +31,7 @@ int str_read_fd(struct str *str, int fd); struct process { struct str stdout; struct str stderr; + struct str both; int fd_io; int fd_er; @@ -43,16 +44,60 @@ struct process* create_process(char *const command[], const char *path); void delete_process(struct process *process); int process_pid(struct process *process); +/** Sends n bytes to process. + +returns the number of bytes written, or -1 indicating an error occurred. An +error will typically occur when the operating system cannot send all n bytes +because the PTY buffer is full. The process will have to read the buffer in +to make space for more data to be written. + +These functions read from the process STDOUT and STDERR buffers to keep +the process from being blocked. The results are buffered and will be returned +on the next call to a process_receive function. If these functions are used +in a separate thread from the process_receive functions, a race condition may +occur when this function is reading from STDOUT/STDERR (possibly realloc'ing) +while `process-receive_*` is reading from the same buffer. +*/ ssize_t process_write(struct process *process, const char* buf, size_t n); ssize_t process_write_string(struct process *process, const char *string); -/** receive process stdout. MUST FREE RETURNED PONTER */ -char* process_receive_stdout(struct process *process); -/** receive process stderr. MUST FREE RETURNED PONTER */ -char* process_receive_stderr(struct process *process); -/** receive process stdout and stderr (one after another). -MUST FREE RETURNED PONTER */ -char* process_receive_output(struct process *process); +/** Sends n bytes to process. + +returns the number of bytes written, or -1 indicating an error occurred. An +error will typically occur when the operating system cannot send all n bytes +because the PTY buffer is full. The process will have to read the buffer in +to make space for more data to be written. + +Doesn't read devices STDOUT/STDERR file descriptors. If `process_receive* +functions are not called regularly, the internal PTY buffers may fill and +prevent the attached process from continuing to run. `process_write` and +`process_write_string` prevent this from happening, but their usage requires +other considerations. +*/ +ssize_t process_write_noread(struct process *process, const char* buf, size_t n); +ssize_t process_write_string_noread(struct process *process, const char *string); + +/** Return Process STDOUT. +Returns pointer to a buffer containing data returned by process STDOUT buffer. +this buffer will be overwritten by subsequent calls to this function; if +this output is meant to be kept, it should be copied out. +*/ +const char* process_receive_stdout(struct process *process); + +/** Return Process STDERR. +Returns pointer to a buffer containing data returned by process STDERR buffer. +this buffer will be overwritten by subsequent calls to this function; if +this output is meant to be kept, it should be copied out. +*/ +const char* process_receive_stderr(struct process *process); + +/** Receive Process STDOUT and STDERR (one after another). +Returns pointer to a buffer containing data returned by process STDERR and +STDOUT buffer. this buffer will be overwritten by subsequent calls to this + function; if this output is meant to be kept, it should be copied out. +*/ +const char* process_receive_output(struct process *process); + int process_alive_p(struct process *process); #endif diff --git a/src/test.c b/src/test.c index 765babd..37e6ecb 100644 --- a/src/test.c +++ b/src/test.c @@ -7,25 +7,25 @@ int main() { struct str s; init_str(&s); - #define TEST_INPUT_SIZE 50000 + #define TEST_INPUT_SIZE 500000 char test_input[TEST_INPUT_SIZE]; - for (size_t i = 0; i < TEST_INPUT_SIZE; i++) { - test_input[i] = '0' + (i % 10); + for (size_t i = 0; i < TEST_INPUT_SIZE; i+=10) { + memcpy(test_input+i, "123456789\n", 10); } size_t n = 0; while (n != TEST_INPUT_SIZE) { ssize_t bytes = process_write(p, test_input+n, TEST_INPUT_SIZE-n); if (bytes > 0) { - printf("writing %d/%d bytes...\n", bytes, n); + printf("wrote %d/%d bytes...\n", n, TEST_INPUT_SIZE); n += bytes; } else { - printf("error: %s\n", strerror(errno)); + printf("%s: %s\n", strerrorname_np(errno), strerror(errno)); } } - + printf("I just attempted to write %d.\nI wrote %d bytes.\n", TEST_INPUT_SIZE, n); while (true) { @@ -37,20 +37,18 @@ int main() { break; } - char *out = NULL; - char *err = NULL; + const char *out = NULL; + const char *err = NULL; out = process_receive_stdout(p); err = process_receive_stderr(p); if (out != NULL) { printf("%s", out); - free(out); } if (err != NULL) { printf("\033[31m%s\033[0m", err); - free(err); } } From 6f3f9369499393100341be176679b66bd17c4dce Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 22 Jan 2026 16:33:44 -0500 Subject: [PATCH 06/17] fixed null terminator bug in receive-output functions --- src/async-process.c | 61 +++++++++++++++++++++++------------------- src/async-process.h | 8 +++--- src/async-process.lisp | 60 +++++++++++++++++++---------------------- src/test.lisp | 31 +++++++++++++++++++++ 4 files changed, 96 insertions(+), 64 deletions(-) create mode 100644 src/test.lisp diff --git a/src/async-process.c b/src/async-process.c index abadc17..771bbe0 100644 --- a/src/async-process.c +++ b/src/async-process.c @@ -92,7 +92,8 @@ void my_exit(int status) { // respectively. Name will be malloced and it is the callers responsibility // to free name. On success, return 0. On fail, returns -1. All references // will also be either initialized or set -1/NULL appropriately. -int open_pty(int *fdm, int *fds, char **name) { +// nonblock will set nonblock mode on the master PTY FD if nonblock == true +int open_pty(int *fdm, int *fds, char **name, bool nonblock) { *fdm = -1; *fds = -1; *name = NULL; @@ -125,7 +126,9 @@ int open_pty(int *fdm, int *fds, char **name) { fcntl(*fds, F_SETFD, FD_CLOEXEC); // set master as non-blocking (for get_process_output functions) - fcntl(*fdm, F_SETFL, O_NONBLOCK); + if (nonblock) { + fcntl(*fdm, F_SETFL, O_NONBLOCK); + } // Set raw mode struct termios tty; @@ -146,7 +149,7 @@ int open_pty(int *fdm, int *fds, char **name) { return -1; } -struct process* create_process(char *const command[], const char *path) { +struct process* create_process(char *const command[], const char *path, bool nonblock) { // Unix PTYs are bi-directional communication streams. Typically, a terminal will // combine stdout and stderr and display them in the same output. We want to // keep the outputs separate at this level so master_pty_er is created just to carry @@ -159,11 +162,11 @@ struct process* create_process(char *const command[], const char *path) { char *pts_io_name, *pts_er_name; int ret; - ret = open_pty(&master_pty_io, &slave_pts_io, &pts_io_name); + ret = open_pty(&master_pty_io, &slave_pts_io, &pts_io_name, nonblock); if (ret == -1) goto FAILED_SETUP; - ret = open_pty(&master_pty_er, &slave_pts_er, &pts_er_name); + ret = open_pty(&master_pty_er, &slave_pts_er, &pts_er_name, nonblock); if (ret == -1) goto FAILED_SETUP; @@ -254,13 +257,13 @@ int str_read_fd(struct str *str, int fd) { str->cap *= 2; } - // read as much data from fd as possible. + // read as much data from fd as possible. (read doesn't add '\0') int n = read(fd, str->buf + str->len, str->cap - str->len); if (total_read == 0 && n == -1) { return -1; // an error occured on first read } else if (n <= 0) { - str->buf[str->len] = '\0'; + str->buf[str->len] = '\0'; // cap is always len+1 return total_read; } @@ -271,49 +274,51 @@ int str_read_fd(struct str *str, int fd) { return -1; // control flow shouldn't reach here. } -const char* _process_receive_fd(struct str *s, int fd) { +const char* _process_receive_fd(struct str *s, int fd, size_t *bytes) { int n = str_read_fd(s, fd); - if (n == -1) + + if (n == -1 || s->len == 0) return NULL; + if (bytes != NULL) + *bytes = s->len; // length of str, not including '\0' return s->buf; } -const char* process_receive_stdout(struct process *p) { - const char *r = _process_receive_fd(&p->stdout, p->fd_io); +const char* process_receive_stdout(struct process *p, size_t *bytes) { + const char *r = _process_receive_fd(&p->stdout, p->fd_io, bytes); p->stdout.len = 0; return r; } -const char* process_receive_stderr(struct process *p) { - const char *r = _process_receive_fd(&p->stderr, p->fd_er); +const char* process_receive_stderr(struct process *p, size_t *bytes) { + const char *r = _process_receive_fd(&p->stderr, p->fd_er, bytes); p->stderr.len = 0; return r; } -const char* process_receive_output(struct process *p) { - _process_receive_fd(&p->stdout, p->fd_io); - _process_receive_fd(&p->stderr, p->fd_er); - +const char* process_receive_output(struct process *p, size_t *bytes) { // these lengths include null terminators - size_t stdout_len = p->stdout.len; - size_t stderr_len = p->stderr.len; + size_t stdout_len = 0; + size_t stderr_len = 0; + + _process_receive_fd(&p->stdout, p->fd_io, &stdout_len); + _process_receive_fd(&p->stderr, p->fd_er, &stderr_len); - if (p->both.cap < (stdout_len + stderr_len)) { + if (p->both.cap < (stdout_len + stderr_len + 1)) { char *new_ptr = realloc(p->both.buf, stdout_len + stderr_len); if (new_ptr == NULL) return NULL; p->both.buf = new_ptr; p->both.cap = stdout_len + stderr_len; } - - // don't copy null terminator - if (stdout_len > 0 && p->stdout.buf[stdout_len] == '\0') { - stdout_len--; - } memcpy(p->both.buf, p->stdout.buf, stdout_len); - memcpy(p->both.buf + stdout_len, p->stderr.buf, stderr_len); // copy null terminator this time. + memcpy(p->both.buf + stdout_len, p->stderr.buf, stderr_len); + p->both.buf[stdout_len + stderr_len] = '\0'; + + if (bytes != NULL) + *bytes = stdout_len + stderr_len; p->both.len = 0; p->stdout.len = 0; @@ -329,8 +334,8 @@ ssize_t _process_write(struct process *process, const char *buf, size_t n, bool ssize_t sent = write(process->fd_io, buf + bytes_written, n - bytes_written); if (readp) { - _process_receive_fd(&process->stdout, process->fd_io); - _process_receive_fd(&process->stderr, process->fd_er); + _process_receive_fd(&process->stdout, process->fd_io, NULL); + _process_receive_fd(&process->stderr, process->fd_er, NULL); } if (bytes_written != 0 && sent == -1) { diff --git a/src/async-process.h b/src/async-process.h index d5467c0..8ca5974 100644 --- a/src/async-process.h +++ b/src/async-process.h @@ -40,7 +40,7 @@ struct process { pid_t pid; }; -struct process* create_process(char *const command[], const char *path); +struct process* create_process(char *const command[], const char *path, bool nonblock); void delete_process(struct process *process); int process_pid(struct process *process); @@ -82,21 +82,21 @@ Returns pointer to a buffer containing data returned by process STDOUT buffer. this buffer will be overwritten by subsequent calls to this function; if this output is meant to be kept, it should be copied out. */ -const char* process_receive_stdout(struct process *process); +const char* process_receive_stdout(struct process *process, size_t *bytes); /** Return Process STDERR. Returns pointer to a buffer containing data returned by process STDERR buffer. this buffer will be overwritten by subsequent calls to this function; if this output is meant to be kept, it should be copied out. */ -const char* process_receive_stderr(struct process *process); +const char* process_receive_stderr(struct process *process, size_t *bytes); /** Receive Process STDOUT and STDERR (one after another). Returns pointer to a buffer containing data returned by process STDERR and STDOUT buffer. this buffer will be overwritten by subsequent calls to this function; if this output is meant to be kept, it should be copied out. */ -const char* process_receive_output(struct process *process); +const char* process_receive_output(struct process *process, size_t *bytes); int process_alive_p(struct process *process); diff --git a/src/async-process.lisp b/src/async-process.lisp index 3d104e9..d7ad235 100644 --- a/src/async-process.lisp +++ b/src/async-process.lisp @@ -57,7 +57,8 @@ (cffi:defcfun ("create_process" %create-process) :pointer (command :pointer) - (path :string)) + (path :string) + (noblock :bool)) (cffi:defcfun ("delete_process" %delete-process) :void (process :pointer)) @@ -65,30 +66,33 @@ (cffi:defcfun ("process_pid" %process-pid) :int (process :pointer)) -(cffi:defcfun ("process_write" %process-write) :void +(cffi:defcfun ("process_write" %process-write) :ssize (process :pointer) (string :string) (n :size)) -(cffi:defcfun ("process_write_string" %process-write-string) :void +(cffi:defcfun ("process_write_string" %process-write-string) :ssize (process :pointer) (string :string)) -(cffi:defcfun ("process_receive_stdout" %process-receive-stdout) :pointer - (process :pointer)) +(cffi:defcfun ("process_receive_stdout" %process-receive-stdout) :string + (process :pointer) + (bytes :pointer)) -(cffi:defcfun ("process_receive_stderr" %process-receive-stderr) :pointer - (process :pointer)) +(cffi:defcfun ("process_receive_stderr" %process-receive-stderr) :string + (process :pointer) + (bytes :pointer)) (cffi:defcfun ("process_receive_output" %process-receive-output) :pointer - (process :pointer)) + (process :pointer) + (bytes :pointer)) (cffi:defcfun ("process_alive_p" %process-alive-p) :boolean (process :pointer)) (cffi:defcfun "cffi_test" :string) -(defun create-process (command &key (encode cffi:*default-foreign-encoding*) directory) +(defun create-process (command &key nonblock (encode cffi:*default-foreign-encoding*) directory) (when (and directory (not (uiop:directory-exists-p directory))) (error "Directory ~S does not exist" directory)) (let* ((command (uiop:ensure-list command)) @@ -98,9 +102,11 @@ :for c :in command :do (setf (cffi:mem-aref argv :string i) c)) (setf (cffi:mem-aref argv :string length) (cffi:null-pointer)) - (let ((p (%create-process argv (if directory - (namestring directory) - (cffi:null-pointer))))) + (let ((p (%create-process argv + (if directory + (namestring directory) + (cffi:null-pointer)) + nonblock))) (if (cffi:null-pointer-p p) (error "create-process failed: ~S" command) (make-instance 'process :process p :encode encode)))))) @@ -115,30 +121,20 @@ (let ((cffi:*default-foreign-encoding* (process-encode process))) (%process-write-string (process-process process) string))) -(defun pointer-to-string (pointer) - (unless (cffi:null-pointer-p pointer) - (let* ((bytes (loop :for i :from 0 - :for code := (cffi:mem-aref pointer :unsigned-char i) - :until (zerop code) - :collect code)) - (octets (make-array (length bytes) - :element-type '(unsigned-byte 8) - :initial-contents bytes))) - (handler-case (babel:octets-to-string octets) - (error () - ;; Fallback when an error occurs with UTF-8 encoding - (map 'string #'code-char octets)))))) - (defun process-receive-output (process &optional (source :both)) - "`source` can be either `:stdout`, `:stderr`, or `:both`. It specifies the stream + "`source` can be either `:stdout`, `:stderr`, or `:both`. It specifies the stream to read from." + (declare (optimize (debug 3))) (flet ((call-cfun (read-func) "helper function to call one of the three cffi functions for receiving output." - (let ((cffi:*default-foreign-encoding* (process-encode process)) - (output (funcall read-func (process-process process)))) - (prog1 - (pointer-to-string output) - (cffi:foreign-free output))))) + (cffi:with-foreign-pointer (bytes 8) + (let ((cffi:*default-foreign-encoding* (process-encode process)) + (output (funcall read-func + (process-process process) + bytes))) + (cffi:foreign-string-to-lisp + output + :count (cffi:mem-ref bytes :size)))))) (case source (:stdout (call-cfun '%process-receive-stdout)) diff --git a/src/test.lisp b/src/test.lisp new file mode 100644 index 0000000..859f36b --- /dev/null +++ b/src/test.lisp @@ -0,0 +1,31 @@ +asdf:*central-registry* +(ql:quickload "alexandria") +(ql:quickload "babel") + +(setf asdf:*central-registry* (list #P"/home/ethan/Documents/async-process/src/")) +(asdf:load-asd #P"/home/ethan/Documents/async-process/src/async-process.asd") + +(asdf:load-system "async-process") + +(defvar *proc* nil) +(setf *proc* (async-process:create-process '("tee" "/home/ethan/test.log") + :nonblock t)) + +(format t "~&~a" + (with-output-to-string (s) + (async-process:process-send-input *proc* (format nil "ima bot~%")) + (sleep 0.1) + (format s "~A" (async-process:process-receive-output *proc*)))) + +(async-process:process-send-input *proc* (format nil "bop~%")) +(format t (async-process:process-receive-output *proc* :both)) + +(format t "~&~S" (async-process:process-receive-output *proc* :both)) + +(defun cffi-null-string-test () + (format t "~&~S" + (cffi:with-pointer-to-vector-data + (p (make-array 10 + :element-type '(unsigned-byte 8) + :initial-element 0)) + (cffi:foreign-string-to-lisp p)))) \ No newline at end of file From 5e3b447e661f17777a36af10d461150ddad1ab3b Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Fri, 23 Jan 2026 13:48:44 -0500 Subject: [PATCH 07/17] removed testing library path --- src/async-process.lisp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/async-process.lisp b/src/async-process.lisp index d7ad235..03d6b53 100644 --- a/src/async-process.lisp +++ b/src/async-process.lisp @@ -49,7 +49,7 @@ (:unix "libasyncprocess.so") (:windows "libasyncprocess.dll")) -(cffi:use-foreign-library #P"/home/ethan/Documents/async-process/.libs/libasyncprocess.so") +; (cffi:use-foreign-library #P"/home/ethan/Documents/async-process/.libs/libasyncprocess.so") (defclass process () ((process :reader process-process :initarg :process) From 7f20a7fa9e6c051d04c3ac5d0ee0bf5250d1aa23 Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Thu, 2 Apr 2026 01:51:51 -0400 Subject: [PATCH 08/17] tranistioning away from C library to pure lisp/cffi implementation. Wrote function to open a pty --- src/async-process-lib.lisp | 154 +++++++++++++++++++++++++++++++++++ src/async-process.lisp | 27 ++++-- src/libc-symbols-grovel.lisp | 28 +++++++ src/test.lisp | 6 +- 4 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 src/async-process-lib.lisp create mode 100644 src/libc-symbols-grovel.lisp diff --git a/src/async-process-lib.lisp b/src/async-process-lib.lisp new file mode 100644 index 0000000..48a47c3 --- /dev/null +++ b/src/async-process-lib.lisp @@ -0,0 +1,154 @@ +(in-package async-process) +(defpackage async-process/libc) + +(cffi:define-foreign-library libc (:default "libc")) +(cffi:use-foreign-library libc) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; TEMP, for debugging. This will be handled by ASDF at load time. +(load (cffi-grovel:process-grovel-file + "/home/ethan/Documents/async-process/src/libc-symbols-grovel.lisp")) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(cffi:defcfun ("setsid" %setsid) :void) +(cffi:defcfun ("execvp" %execvp) :void) +(cffi:defcfun ("dup2" %dup2) :void) +(cffi:defcfun ("fork" %fork) :void) + +(cffi:defcfun ("open" %open) :int + (pathname :string) + (flags :int) + &rest) + +(cffi:defcfun ("close" %close) :int + (fd :int)) + +(cffi:defcfun ("posix_openpt" %posix_openpt) :int + (flags :int)) + +(cffi:defcfun ("grantpt" %grantpt) :int + (master_fd :int)) + +(cffi:defcfun ("unlockpt" %unlockpt) :int + (master_fd :int)) + +(cffi:defcfun ("ptsname" %ptsname) :string + "The C function returns a char *, which must be copied. Here, +CFFI automaticaly does this conversion/copy for us." + (master_fd :int)) + +(cffi:defcfun ("fcntl" %fcntl) :int + (filedes :int) + (cmd :int) + &rest) + +;; termios struct retrieved by groveler +(cffi:defcfun ("tcgetattr" %tcgetattr) :int + (fd :int) + (termios-ptr (:pointer (:struct termios)))) + +;; termios struct retreived by groveler +(cffi:defcfun ("tcsetattr" %tcsetattr) :int + (fd :int) + (optional_actions :int) + (termios-ptr (:pointer (:struct termios)))) + +;; termios struct retreived by groveler +(cffi:defcfun ("cfmakeraw" %cfmakeraw) :void + (termios-ptr (:pointer (:struct termios)))) + +(defstruct posix-pty + (fdm) + (fds) + (name)) + +(defun init-pty (fdm fds &optional nonblock) + "helper function to set file attributes/settings on master/slave file +descriptors after they are opened." + ;; ensure both slave and master close after program finishes + (%fcntl fdm +f-setfd+ :int +fd-cloexec+) + (%fcntl fds +f-setfd+ :int +fd-cloexec+) + + ;; set master as non-blocking + (when nonblock + (%fcntl fdm +f-setfl+ :int +o_nonblock+)) + + ;; set raw mode + (cffi:with-foreign-object (tty '(:struct termios)) + (%tcgetattr fds tty) + (%cfmakeraw tty) + (%tcsetattr fds +tcsanow+ tty)) + + ;; always return nil + nil) + +(defun close-pty (pty) + (declare (type posix-pty pty)) + (let ((fdm (posix-pty-fdm pty)) + (fds (posix-pty-fds pty))) + (when (/= -1 fdm) (%close fdm)) + (setf (posix-pty-fdm pty) -1) + + (when (/= -1 fds) (%close fds)) + (setf (posix-pty-fds pty) -1) + + (setf (posix-pty-name pty) "")) + nil) + + +(defun open-pty (&optional nonblock) + "opens a PTY and returns a `POSIX-PTY` struct." + (declare (optimize (debug 3))) + + ;; get the master FD through `posix_openpt`. + (let* ((open-flags (logior +o-rdwr+ +o-noctty+)) + (fdm (%posix_openpt open-flags)) + (fds -1) + (name nil) + (pty nil)) + + (when + (prog1 + (cond + ;; Initialized the attached slave PTS. grantpt and unlockpt + ;; are required before opening the slave device. + ((or (eq -1 fdm) + (eq -1 (%grantpt fdm)) + (eq -1 (%unlockpt fdm)))) + + ;; get pathname of pty we just opened. + ((eq nil (setf name (%ptsname fdm))) + (equal name nil)) + + ;; use pathname to open slave file descriptor + ((eq -1 (setf fds (%open name open-flags)))) + + ;; both fds and fdm are opened, finish configuration + ((init-pty fdm fds nonblock))) + (setf pty (make-posix-pty :fdm fdm :fds fds :name name))) + + ;; if any of the conditions were true, then there was an error somewhere. + (format t "WARNING: couldn't open PTY") + (close-pty pty)) + + pty)) + +(defun pty-test (test-str) + "should print " + (let ((test (open-pty t))) + (cffi:with-foreign-string ((buf len) test-str) + (when (= -1 (cffi:foreign-funcall "write" + :int (posix-pty-fdm test) + :pointer buf + :size (1- len) + :ssize)) + (cffi:foreign-funcall "perror" :string "write error" :void))) + + (cffi:with-foreign-object (buf :uint8 1024) + (let ((n (cffi:foreign-funcall "read" :int (posix-pty-fds test) :pointer buf :size 1024 :ssize))) + (when (> n 0) + (format t "got ~A: ~A" n (cffi:foreign-string-to-lisp buf :count n))))) + + (close-pty test))) + +(pty-test "hello world") diff --git a/src/async-process.lisp b/src/async-process.lisp index 03d6b53..42e0cb8 100644 --- a/src/async-process.lisp +++ b/src/async-process.lisp @@ -19,9 +19,12 @@ :ignore-error-status t :output :string))))))) +(format t "~&async process is at: ~A" + (asdf:system-relative-pathname "async-process" ".")) + (pushnew (asdf:system-relative-pathname :async-process - (format nil "../static/~A/" + (format nil "../.libs/~A/" (cond ;; Windows ((uiop/os:featurep '(:and :windows :x86-64)) @@ -38,18 +41,24 @@ (uiop:run-program '("uname" "-m") :output '(:string :stripped t)) (let ((os (uiop:run-program '("uname") :output '(:string :stripped t)))) (cond ((and (equal os "Linux") - (ignore-errors (funcall (read-from-string "muslp")))) + (ignore-errors (funcall (read-from-string "muslp"))))n "Linux-musl") (t os)))))))) cffi:*foreign-library-directories* :test #'uiop:pathname-equal) -(cffi:define-foreign-library async-process - (:darwin "libasyncprocess.dylib") - (:unix "libasyncprocess.so") - (:windows "libasyncprocess.dll")) +(pushnew (asdf:system-relative-pathname "async-process" "../.libs/") cffi:*foreign-library-directories*) + +;; this binds the library file path to the symbol async-process for use with +;; `use-foreign-library` +(format t "~&cffi foreign library status: ~A" + (cffi:define-foreign-library async-process + (:darwin "libasyncprocess.dylib") + (:unix "libasyncprocess.so") + (:windows "libasyncprocess.dll"))) -; (cffi:use-foreign-library #P"/home/ethan/Documents/async-process/.libs/libasyncprocess.so") +;; This currently fails because qlot does not copy the .libs folder over. +;;(cffi:use-foreign-library async-process) (defclass process () ((process :reader process-process :initarg :process) @@ -75,11 +84,11 @@ (process :pointer) (string :string)) -(cffi:defcfun ("process_receive_stdout" %process-receive-stdout) :string +(cffi:defcfun ("process_receive_stdout" %process-receive-stdout) :pointer (process :pointer) (bytes :pointer)) -(cffi:defcfun ("process_receive_stderr" %process-receive-stderr) :string +(cffi:defcfun ("process_receive_stderr" %process-receive-stderr) :pointer (process :pointer) (bytes :pointer)) diff --git a/src/libc-symbols-grovel.lisp b/src/libc-symbols-grovel.lisp new file mode 100644 index 0000000..0dee6fb --- /dev/null +++ b/src/libc-symbols-grovel.lisp @@ -0,0 +1,28 @@ +(in-package async-process) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(include "fcntl.h") +(constant (+o-rdwr+ "O_RDWR")) +(constant (+o-noctty+ "O_NOCTTY")) +(constant (+o_nonblock+ "O_NONBLOCK")) + +(constant (+f-setfd+ "F_SETFD")) +(constant (+f-setfl+ "F_SETFL")) + +(constant (+fd-cloexec+ "FD_CLOEXEC")) + +(constant (+tcsanow+ "TCSANOW")) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(include "unistd.h") +(constant (+stdin-fileno+ "STDIN_FILENO")) +(constant (+stdout-fileno+ "STDOUT_FILENO")) +(constant (+stderr-fileno+ "STDERR_FILENO")) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(include "errno.h") +(constant (+enoent+ "ENOENT")) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(include "termios.h") +(cstruct termios "struct termios") diff --git a/src/test.lisp b/src/test.lisp index 859f36b..5b02724 100644 --- a/src/test.lisp +++ b/src/test.lisp @@ -28,4 +28,8 @@ asdf:*central-registry* (p (make-array 10 :element-type '(unsigned-byte 8) :initial-element 0)) - (cffi:foreign-string-to-lisp p)))) \ No newline at end of file + (cffi:foreign-string-to-lisp p)))) + +;;; test which cffi lib is currently loaded. +(format t "~&---~%~{~A~%~}" (cffi:list-foreign-libraries)) +(format t "~A" (cffi:foreign-library-pathname 'async-process::async-process)) From 5d541d6b8aeefa5ceae526c1aa40f223b1bf75ff Mon Sep 17 00:00:00 2001 From: Ethan Smith Date: Sun, 5 Apr 2026 09:50:01 -0400 Subject: [PATCH 09/17] syncing commit, WIP --- src/async-process-lib.lisp | 108 ++++++++++++++++++++++++++++--------- src/async-process.asd | 1 + 2 files changed, 85 insertions(+), 24 deletions(-) diff --git a/src/async-process-lib.lisp b/src/async-process-lib.lisp index 48a47c3..817e717 100644 --- a/src/async-process-lib.lisp +++ b/src/async-process-lib.lisp @@ -1,15 +1,21 @@ (in-package async-process) -(defpackage async-process/libc) (cffi:define-foreign-library libc (:default "libc")) (cffi:use-foreign-library libc) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; TEMP, for debugging. This will be handled by ASDF at load time. +;;; TEMP, for debugging. This will be handled by ASDF at load time. (load (cffi-grovel:process-grovel-file "/home/ethan/Documents/async-process/src/libc-symbols-grovel.lisp")) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Libc Function Definitions +;;; +;;; these are the libc functions that are used in this program to +;;; open/manage the PTY's for each process. The CFFI Groveler is also used +;;; to pull in constants/struct definitions used as parameters. + (cffi:defcfun ("setsid" %setsid) :void) (cffi:defcfun ("execvp" %execvp) :void) (cffi:defcfun ("dup2" %dup2) :void) @@ -57,6 +63,12 @@ CFFI automaticaly does this conversion/copy for us." (cffi:defcfun ("cfmakeraw" %cfmakeraw) :void (termios-ptr (:pointer (:struct termios)))) +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; PTY specific functions +;;; +;;; The following functions are used to open/close a pty. The posix-pty +;;; struct is used to hold and keep track of the state of the PTY's + (defstruct posix-pty (fdm) (fds) @@ -83,6 +95,8 @@ descriptors after they are opened." nil) (defun close-pty (pty) + "This must be called on the PTY if the program doesn't terminate +TODO: figure out the exact semantics of when this needs to be called." (declare (type posix-pty pty)) (let ((fdm (posix-pty-fdm pty)) (fds (posix-pty-fds pty))) @@ -107,31 +121,77 @@ descriptors after they are opened." (name nil) (pty nil)) - (when - (prog1 - (cond - ;; Initialized the attached slave PTS. grantpt and unlockpt - ;; are required before opening the slave device. - ((or (eq -1 fdm) - (eq -1 (%grantpt fdm)) - (eq -1 (%unlockpt fdm)))) + (if (cond + ;; Initialized the attached slave PTS. grantpt and unlockpt + ;; are required before opening the slave device. + ((or (eq -1 fdm) + (eq -1 (%grantpt fdm)) + (eq -1 (%unlockpt fdm)))) - ;; get pathname of pty we just opened. - ((eq nil (setf name (%ptsname fdm))) - (equal name nil)) + ;; get pathname of pty we just opened. + ((eq nil (setf name (%ptsname fdm))) + (equal name nil)) - ;; use pathname to open slave file descriptor - ((eq -1 (setf fds (%open name open-flags)))) + ;; use pathname to open slave file descriptor + ((eq -1 (setf fds (%open name open-flags)))) - ;; both fds and fdm are opened, finish configuration - ((init-pty fdm fds nonblock))) - (setf pty (make-posix-pty :fdm fdm :fds fds :name name))) - - ;; if any of the conditions were true, then there was an error somewhere. - (format t "WARNING: couldn't open PTY") - (close-pty pty)) - - pty)) + ;; both fds and fdm are opened, finish configuration + ((and (init-pty fdm fds nonblock) + nil)) ; the and ensures that nil is returned + ((and (setf pty (make-posix-pty :fdm fdm :fds fds :name name)) + nil))) ; the and ensures that nil is returned + (progn + ;; if any of the conditions were true, then there was an error somewhere. + ;; cleanup and return NIL + (format t "WARNING: couldn't open PTY") + (close-pty pty) + nil) + + ;; no errors occured, return the PTY + pty))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Process specific functions +;;; +;;; Unix PTYs are bi-diretional communication streams. Typically, a +;;; terminal will combine STDOUT/STDERR and display them in the same output. +;;; We want to keep the outputs separate at this level so that the process +;;; can send output on separate STDOUT and STDERR streams. +;;; +;;; Note that this introduces a potential bug: if the process tries to set +;;; terminal attributes (like with stty), these updates won't be propogated +;;; across both STDOUT and STDERR terminals. + +(defstruct (posix-process (:constructor nil)) + (stdio-pty) + (stder-pty) + (command) + (path) + (nonblockp)) + +(defun make-posix-process (command path nonblockp) + "creates a posix-process struct. Doesn't open any PTYs or start any +programs. This is purely to initialize the `posix-process` structure." + (make-instance 'posix-process + :stdio-pty nil + :stder-pty nil + :command command + :path path + :nonblockp nonblockp)) + +(defun start-process (proc) + (let (())) + (cond + ((not (setf (posix-process-stdio-pty proc) (open-pty )))))) + +(defun close-process (posix-process)) + + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; Testing Functions +;;; +;;; used to validate functionality and correctness of this library. (defun pty-test (test-str) "should print " diff --git a/src/async-process.asd b/src/async-process.asd index 9f506ec..e7070be 100644 --- a/src/async-process.asd +++ b/src/async-process.asd @@ -8,4 +8,5 @@ :components ((:file "async-process_windows" :if-feature (:or :win32 :windows)) (:file "async-process" + :cffi-grovel-file "libc-symbols-grovel" :if-feature (:not (:or :win32 :windows))))) From c476af58971712855c7843542f0f20ec60c0972e Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Mon, 6 Apr 2026 02:16:21 -0400 Subject: [PATCH 10/17] added proces start function, needs testing, probably has errors. WIP --- src/async-process-lib.lisp | 96 +++++++++++++++++++++++++++++++----- src/libc-symbols-grovel.lisp | 1 + 2 files changed, 85 insertions(+), 12 deletions(-) diff --git a/src/async-process-lib.lisp b/src/async-process-lib.lisp index 817e717..fc645c1 100644 --- a/src/async-process-lib.lisp +++ b/src/async-process-lib.lisp @@ -6,7 +6,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; TEMP, for debugging. This will be handled by ASDF at load time. (load (cffi-grovel:process-grovel-file - "/home/ethan/Documents/async-process/src/libc-symbols-grovel.lisp")) + "/home/ethanxxxl/Documents/async-process/src/libc-symbols-grovel.lisp")) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -17,9 +17,16 @@ ;;; to pull in constants/struct definitions used as parameters. (cffi:defcfun ("setsid" %setsid) :void) -(cffi:defcfun ("execvp" %execvp) :void) -(cffi:defcfun ("dup2" %dup2) :void) -(cffi:defcfun ("fork" %fork) :void) +(cffi:defcfun ("execvp" %execvp) :int + "executes the program at file, with the argument list argv." + (file :string) + (argv :pointer)) + +(cffi:defcfun ("dup2" %dup2) :int + (oldfd :int) + (newfd :int)) + +(cffi:defcfun ("fork" %fork) :int) (cffi:defcfun ("open" %open) :int (pathname :string) @@ -47,7 +54,7 @@ CFFI automaticaly does this conversion/copy for us." (filedes :int) (cmd :int) &rest) - +:int ;; termios struct retrieved by groveler (cffi:defcfun ("tcgetattr" %tcgetattr) :int (fd :int) @@ -109,7 +116,6 @@ TODO: figure out the exact semantics of when this needs to be called." (setf (posix-pty-name pty) "")) nil) - (defun open-pty (&optional nonblock) "opens a PTY and returns a `POSIX-PTY` struct." (declare (optimize (debug 3))) @@ -179,14 +185,77 @@ programs. This is purely to initialize the `posix-process` structure." :path path :nonblockp nonblockp)) -(defun start-process (proc) - (let (())) - (cond - ((not (setf (posix-process-stdio-pty proc) (open-pty )))))) - -(defun close-process (posix-process)) +(defun close-process (proc) + "terminates the process and closes resources associated with it." + ;; TODO Terminate Process + ;; Close PTYs + (close-pty (posix-process-stdio-pty proc)) + (setf (posix-process-stdio-pty proc) nil) + + (close-pty (posix-process-stder-pty proc)) + (setf (posix-process-stder-pty proc) nil)) + +(defun %start-process (proc) + "handles the creation of a child process after fork is run." + (declare (type posix-process proc)) + (with-slots (stdio-pty stder-pty command path) proc + (%setsid) + (%close (posix-pty-fdm stdio-pty)) + (%close (posix-pty-fdm stder-pty)) + + (%dup2 (posix-pty-fds stdio-pty) +stdin-fileno+) + (%dup2 (posix-pty-fds stdio-pty) +stdout-fileno+) + (%dup2 (posix-pty-fds stder-pty) +stderr-fileno+) + + (if path + (uiop:chdir path)) + + (cffi:with-foreign-array (argv + (make-array (1- (length command)) + :initial-contents (cdr command)) + :string) + (cffi:with-foreign-string (cmd (first command)) + (%execvp cmd argv))) + + ;; If execution reaches here, that means execvp failed in some way. + (let ((e *errno*)) + (if (= e +ENOENT+) + (error "No such file or directory: ~A" (first command))) + + (error (cffi:foreign-funcall "strerror" :int *errno* :string))))) +(defun start-process (proc) + "Creates a new process (using fork), and starts execution +of the program." + (declare (type posix-process proc)) + + (unwind-protect + (let* ((nonblockp (posix-process-nonblockp proc)) + (io-pty (open-pty nonblockp)) + (er-pty (open-pty nonblockp)) + (pid -1)) + + (setf (posix-process-stdio-pty proc) io-pty + (posix-process-stder-pty proc) er-pty) + + (unless (and io-pty er-pty) + (error "failed to open pty")) + + ;; Start child process and return it's PID + (setf pid (%fork)) + (if (= pid -1) + (error "Failed to Fork")) + + ;; this is the parent process, close slave file descriptors + (when (/= pid 0) + (%close (posix-pty-fds io-pty)) + (%close (posix-pty-fds er-pty))) + + ;; this is the child process, initialize and run program + (when (= pid 0) + (%start-process proc))) + (close-process proc))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Testing Functions @@ -212,3 +281,6 @@ programs. This is purely to initialize the `posix-process` structure." (close-pty test))) (pty-test "hello world") + +(let ((p (make-posix-process '("echo" "Hello" "World") nil nil))) + (start-process p)) \ No newline at end of file diff --git a/src/libc-symbols-grovel.lisp b/src/libc-symbols-grovel.lisp index 0dee6fb..6efe2e4 100644 --- a/src/libc-symbols-grovel.lisp +++ b/src/libc-symbols-grovel.lisp @@ -21,6 +21,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (include "errno.h") +(cvar ("errno" *errno*) :int) (constant (+enoent+ "ENOENT")) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; From 375dd29426f9fd9be0a829992b37c57e57c0bcf1 Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Fri, 17 Apr 2026 00:27:22 -0400 Subject: [PATCH 11/17] implemented async-process using uiop:launch-program --- src/async-process-uiop.lisp | 123 ++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 src/async-process-uiop.lisp diff --git a/src/async-process-uiop.lisp b/src/async-process-uiop.lisp new file mode 100644 index 0000000..5a31f56 --- /dev/null +++ b/src/async-process-uiop.lisp @@ -0,0 +1,123 @@ +(in-package async-process) + +(defvar *active-processes* nil + "list of processes started by async-process. If a process won't exit and needs +killed, it can be found in this list.") + +(defclass process () + ((info :type uiop:process-info :initarg :info) + (nonblockp :type t :initform t :initarg :nonblockp)) + (:documentation "Represents an asynchronous process. `async-process` used to +implement bespoke logic for starting processes. Now, this functionality is +implemented using `uiop:launch-program` which returns a `process-info` class")) + +(defun create-process (command &rest keys &key nonblock + (encode cffi:*default-foreign-encoding*) + &allow-other-keys) + "calls creates a process that runs in the background. `DELETE-PROCESS` must +be called when process is completed. Passes arguments to uiop:launch-program. +`NONBLOCK` will affect behavior of reading output. Encode is not used." + (declare (ignore encode)) + + (let ((proc (make-instance 'process + :nonblockp nonblock + :info (apply 'uiop:launch-program + command + :input :stream + :output :stream + keys)))) + + (push proc *active-processes*) + proc)) + +(defun delete-process (proc) + "terminate `PROCESS` and remove it from *active-processes*" + (declare (type process proc)) + + (with-slots (info) proc + (when (uiop:process-alive-p info) + (uiop:terminate-process info) + (uiop:wait-process info))) + + (setf *active-processes* (delete proc *active-processes*)) + t) + +(defun process-pid (proc) + (declare (type process proc)) + + (uiop:process-info-pid (slot-value proc 'info))) + +(defun process-send-input (proc input) + (declare (type process proc) + (type string input)) + + (let ((s (uiop:process-info-input (slot-value proc 'info)))) + (write-string input s) + (finish-output s))) + +(defun process-receive-output (proc) + (declare (type process proc) + (optimize (debug 3))) + + (with-slots (info nonblockp) proc + (let ((s (uiop:process-info-output info))) + (unless (and nonblockp (not (listen s))) + ;; read output, unless we are nonblocking and there is no data available + + (loop :with v = (make-array 20 + :element-type 'character + :adjustable t + :fill-pointer 0) + :for c = (read-char-no-hang s nil nil) + :while c + :do (vector-push-extend c v) + :finally (return v)))))) + + +(defun process-alive-p (proc) + (declare (type process proc)) + + (uiop:process-alive-p (slot-value proc 'info))) + +(defun test-process-output () + (let ((proc (create-process '("echo" "hello" "world") :nonblock nil))) + (sleep 0.5) + (format t "~&ouptut: ~S" (process-receive-output proc)) + (delete-process proc))) + +(defun test-process-input () + (let ((proc (create-process '("tee") :nonblock nil))) + (sleep 0.5) + (process-send-input proc "hello world +") + (format t "~&output: ~S" (process-receive-output proc)) + (delete-process proc))) + +;(test-process-input) + +(defvar *test-proc* nil) + +(defun test1 () + (setf *test-proc* (create-process '("tee") :nonblock t))) + +(defun test2 () + (process-send-input *test-proc* "hello world +")) +(defun test-process-input () + (let ((proc (create-process '("tee") :nonblock nil))) + (sleep 0.5) + (process-send-input proc "hello world +") + (format t "~&output: ~S" (process-receive-output proc)) + (delete-process proc))) +(defun test3 () + (format t "~&~A" (process-receive-output *test-proc*))) + +(defun test1-2-cleanup () + (delete-process *test-proc*) + (setf *test-proc* nil)) + +;(test1) +;(test2) +;(test3) +;(test1-2-cleanup) \ No newline at end of file From 72dc4230e7c3d566ffe2a8fb4b538d38832cb9ab Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Fri, 17 Apr 2026 01:15:52 -0400 Subject: [PATCH 12/17] lem now uses uiop implementation --- src/async-process-uiop.lisp | 16 ++++++++++++++-- src/async-process.asd | 3 +-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/async-process-uiop.lisp b/src/async-process-uiop.lisp index 5a31f56..154edf8 100644 --- a/src/async-process-uiop.lisp +++ b/src/async-process-uiop.lisp @@ -1,3 +1,12 @@ +(defpackage :async-process + (:use :cl) + (:export + :delete-process + :process-send-input + :process-receive-output + :process-alive-p + :create-process)) + (in-package async-process) (defvar *active-processes* nil @@ -6,7 +15,8 @@ killed, it can be found in this list.") (defclass process () ((info :type uiop:process-info :initarg :info) - (nonblockp :type t :initform t :initarg :nonblockp)) + (nonblockp :type t :initform t :initarg :nonblockp) + (command :type string :initarg :command)) (:documentation "Represents an asynchronous process. `async-process` used to implement bespoke logic for starting processes. Now, this functionality is implemented using `uiop:launch-program` which returns a `process-info` class")) @@ -20,11 +30,13 @@ be called when process is completed. Passes arguments to uiop:launch-program. (declare (ignore encode)) (let ((proc (make-instance 'process + :command command :nonblockp nonblock :info (apply 'uiop:launch-program command :input :stream - :output :stream + :output :stream + :error-output :stream keys)))) (push proc *active-processes*) diff --git a/src/async-process.asd b/src/async-process.asd index e7070be..eefc050 100644 --- a/src/async-process.asd +++ b/src/async-process.asd @@ -7,6 +7,5 @@ :serial t :components ((:file "async-process_windows" :if-feature (:or :win32 :windows)) - (:file "async-process" - :cffi-grovel-file "libc-symbols-grovel" + (:file "async-process-uiop" :if-feature (:not (:or :win32 :windows))))) From fa2ca0dcc52b5c31b114f03802f741d507c4fa62 Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Fri, 17 Apr 2026 22:28:28 -0400 Subject: [PATCH 13/17] fixed issue when receiving process output in blocking mode --- src/async-process-uiop.lisp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/async-process-uiop.lisp b/src/async-process-uiop.lisp index 154edf8..9cb63eb 100644 --- a/src/async-process-uiop.lisp +++ b/src/async-process-uiop.lisp @@ -70,20 +70,22 @@ be called when process is completed. Passes arguments to uiop:launch-program. (defun process-receive-output (proc) (declare (type process proc) (optimize (debug 3))) - + (with-slots (info nonblockp) proc - (let ((s (uiop:process-info-output info))) - (unless (and nonblockp (not (listen s))) - ;; read output, unless we are nonblocking and there is no data available - - (loop :with v = (make-array 20 - :element-type 'character - :adjustable t - :fill-pointer 0) - :for c = (read-char-no-hang s nil nil) - :while c - :do (vector-push-extend c v) - :finally (return v)))))) + (let ((s (uiop:process-info-output info)) + (blockp (not nonblockp)) + (str-out (make-array 20 + :element-type 'character + :adjustable t + :fill-pointer 0))) + (when (or blockp (listen s)) + ;; attempt reading output whenever there is data, or we can block + + (loop :for c = (read-char-no-hang s nil nil) + :while (or (and blockp (= 0 (length str-out))) + c) + :do (when c (vector-push-extend c str-out)))) + str-out))) (defun process-alive-p (proc) From ba87d717b6b91143afc2344f317bb840ec57325a Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Fri, 17 Apr 2026 23:00:21 -0400 Subject: [PATCH 14/17] removed superfluous files and cffi dependency --- src/async-process-lib.lisp | 286 --------------------------- src/async-process.asd | 1 - src/async-process.c | 374 ----------------------------------- src/async-process.h | 103 ---------- src/async-process.lisp | 154 --------------- src/libc-symbols-grovel.lisp | 29 --- src/test.c | 58 ------ src/test.lisp | 35 ---- 8 files changed, 1040 deletions(-) delete mode 100644 src/async-process-lib.lisp delete mode 100644 src/async-process.c delete mode 100644 src/async-process.h delete mode 100644 src/async-process.lisp delete mode 100644 src/libc-symbols-grovel.lisp delete mode 100644 src/test.c delete mode 100644 src/test.lisp diff --git a/src/async-process-lib.lisp b/src/async-process-lib.lisp deleted file mode 100644 index fc645c1..0000000 --- a/src/async-process-lib.lisp +++ /dev/null @@ -1,286 +0,0 @@ -(in-package async-process) - -(cffi:define-foreign-library libc (:default "libc")) -(cffi:use-foreign-library libc) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; TEMP, for debugging. This will be handled by ASDF at load time. -(load (cffi-grovel:process-grovel-file - "/home/ethanxxxl/Documents/async-process/src/libc-symbols-grovel.lisp")) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; Libc Function Definitions -;;; -;;; these are the libc functions that are used in this program to -;;; open/manage the PTY's for each process. The CFFI Groveler is also used -;;; to pull in constants/struct definitions used as parameters. - -(cffi:defcfun ("setsid" %setsid) :void) -(cffi:defcfun ("execvp" %execvp) :int - "executes the program at file, with the argument list argv." - (file :string) - (argv :pointer)) - -(cffi:defcfun ("dup2" %dup2) :int - (oldfd :int) - (newfd :int)) - -(cffi:defcfun ("fork" %fork) :int) - -(cffi:defcfun ("open" %open) :int - (pathname :string) - (flags :int) - &rest) - -(cffi:defcfun ("close" %close) :int - (fd :int)) - -(cffi:defcfun ("posix_openpt" %posix_openpt) :int - (flags :int)) - -(cffi:defcfun ("grantpt" %grantpt) :int - (master_fd :int)) - -(cffi:defcfun ("unlockpt" %unlockpt) :int - (master_fd :int)) - -(cffi:defcfun ("ptsname" %ptsname) :string - "The C function returns a char *, which must be copied. Here, -CFFI automaticaly does this conversion/copy for us." - (master_fd :int)) - -(cffi:defcfun ("fcntl" %fcntl) :int - (filedes :int) - (cmd :int) - &rest) -:int -;; termios struct retrieved by groveler -(cffi:defcfun ("tcgetattr" %tcgetattr) :int - (fd :int) - (termios-ptr (:pointer (:struct termios)))) - -;; termios struct retreived by groveler -(cffi:defcfun ("tcsetattr" %tcsetattr) :int - (fd :int) - (optional_actions :int) - (termios-ptr (:pointer (:struct termios)))) - -;; termios struct retreived by groveler -(cffi:defcfun ("cfmakeraw" %cfmakeraw) :void - (termios-ptr (:pointer (:struct termios)))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; PTY specific functions -;;; -;;; The following functions are used to open/close a pty. The posix-pty -;;; struct is used to hold and keep track of the state of the PTY's - -(defstruct posix-pty - (fdm) - (fds) - (name)) - -(defun init-pty (fdm fds &optional nonblock) - "helper function to set file attributes/settings on master/slave file -descriptors after they are opened." - ;; ensure both slave and master close after program finishes - (%fcntl fdm +f-setfd+ :int +fd-cloexec+) - (%fcntl fds +f-setfd+ :int +fd-cloexec+) - - ;; set master as non-blocking - (when nonblock - (%fcntl fdm +f-setfl+ :int +o_nonblock+)) - - ;; set raw mode - (cffi:with-foreign-object (tty '(:struct termios)) - (%tcgetattr fds tty) - (%cfmakeraw tty) - (%tcsetattr fds +tcsanow+ tty)) - - ;; always return nil - nil) - -(defun close-pty (pty) - "This must be called on the PTY if the program doesn't terminate -TODO: figure out the exact semantics of when this needs to be called." - (declare (type posix-pty pty)) - (let ((fdm (posix-pty-fdm pty)) - (fds (posix-pty-fds pty))) - (when (/= -1 fdm) (%close fdm)) - (setf (posix-pty-fdm pty) -1) - - (when (/= -1 fds) (%close fds)) - (setf (posix-pty-fds pty) -1) - - (setf (posix-pty-name pty) "")) - nil) - -(defun open-pty (&optional nonblock) - "opens a PTY and returns a `POSIX-PTY` struct." - (declare (optimize (debug 3))) - - ;; get the master FD through `posix_openpt`. - (let* ((open-flags (logior +o-rdwr+ +o-noctty+)) - (fdm (%posix_openpt open-flags)) - (fds -1) - (name nil) - (pty nil)) - - (if (cond - ;; Initialized the attached slave PTS. grantpt and unlockpt - ;; are required before opening the slave device. - ((or (eq -1 fdm) - (eq -1 (%grantpt fdm)) - (eq -1 (%unlockpt fdm)))) - - ;; get pathname of pty we just opened. - ((eq nil (setf name (%ptsname fdm))) - (equal name nil)) - - ;; use pathname to open slave file descriptor - ((eq -1 (setf fds (%open name open-flags)))) - - ;; both fds and fdm are opened, finish configuration - ((and (init-pty fdm fds nonblock) - nil)) ; the and ensures that nil is returned - ((and (setf pty (make-posix-pty :fdm fdm :fds fds :name name)) - nil))) ; the and ensures that nil is returned - (progn - ;; if any of the conditions were true, then there was an error somewhere. - ;; cleanup and return NIL - (format t "WARNING: couldn't open PTY") - (close-pty pty) - nil) - - ;; no errors occured, return the PTY - pty))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; Process specific functions -;;; -;;; Unix PTYs are bi-diretional communication streams. Typically, a -;;; terminal will combine STDOUT/STDERR and display them in the same output. -;;; We want to keep the outputs separate at this level so that the process -;;; can send output on separate STDOUT and STDERR streams. -;;; -;;; Note that this introduces a potential bug: if the process tries to set -;;; terminal attributes (like with stty), these updates won't be propogated -;;; across both STDOUT and STDERR terminals. - -(defstruct (posix-process (:constructor nil)) - (stdio-pty) - (stder-pty) - (command) - (path) - (nonblockp)) - -(defun make-posix-process (command path nonblockp) - "creates a posix-process struct. Doesn't open any PTYs or start any -programs. This is purely to initialize the `posix-process` structure." - (make-instance 'posix-process - :stdio-pty nil - :stder-pty nil - :command command - :path path - :nonblockp nonblockp)) - -(defun close-process (proc) - "terminates the process and closes resources associated with it." - ;; TODO Terminate Process - - ;; Close PTYs - (close-pty (posix-process-stdio-pty proc)) - (setf (posix-process-stdio-pty proc) nil) - - (close-pty (posix-process-stder-pty proc)) - (setf (posix-process-stder-pty proc) nil)) - -(defun %start-process (proc) - "handles the creation of a child process after fork is run." - (declare (type posix-process proc)) - (with-slots (stdio-pty stder-pty command path) proc - (%setsid) - (%close (posix-pty-fdm stdio-pty)) - (%close (posix-pty-fdm stder-pty)) - - (%dup2 (posix-pty-fds stdio-pty) +stdin-fileno+) - (%dup2 (posix-pty-fds stdio-pty) +stdout-fileno+) - (%dup2 (posix-pty-fds stder-pty) +stderr-fileno+) - - (if path - (uiop:chdir path)) - - (cffi:with-foreign-array (argv - (make-array (1- (length command)) - :initial-contents (cdr command)) - :string) - (cffi:with-foreign-string (cmd (first command)) - (%execvp cmd argv))) - - ;; If execution reaches here, that means execvp failed in some way. - (let ((e *errno*)) - (if (= e +ENOENT+) - (error "No such file or directory: ~A" (first command))) - - (error (cffi:foreign-funcall "strerror" :int *errno* :string))))) - -(defun start-process (proc) - "Creates a new process (using fork), and starts execution -of the program." - (declare (type posix-process proc)) - - (unwind-protect - (let* ((nonblockp (posix-process-nonblockp proc)) - (io-pty (open-pty nonblockp)) - (er-pty (open-pty nonblockp)) - (pid -1)) - - (setf (posix-process-stdio-pty proc) io-pty - (posix-process-stder-pty proc) er-pty) - - (unless (and io-pty er-pty) - (error "failed to open pty")) - - ;; Start child process and return it's PID - (setf pid (%fork)) - (if (= pid -1) - (error "Failed to Fork")) - - ;; this is the parent process, close slave file descriptors - (when (/= pid 0) - (%close (posix-pty-fds io-pty)) - (%close (posix-pty-fds er-pty))) - - ;; this is the child process, initialize and run program - (when (= pid 0) - (%start-process proc))) - (close-process proc))) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;;; Testing Functions -;;; -;;; used to validate functionality and correctness of this library. - -(defun pty-test (test-str) - "should print " - (let ((test (open-pty t))) - (cffi:with-foreign-string ((buf len) test-str) - (when (= -1 (cffi:foreign-funcall "write" - :int (posix-pty-fdm test) - :pointer buf - :size (1- len) - :ssize)) - (cffi:foreign-funcall "perror" :string "write error" :void))) - - (cffi:with-foreign-object (buf :uint8 1024) - (let ((n (cffi:foreign-funcall "read" :int (posix-pty-fds test) :pointer buf :size 1024 :ssize))) - (when (> n 0) - (format t "got ~A: ~A" n (cffi:foreign-string-to-lisp buf :count n))))) - - (close-pty test))) - -(pty-test "hello world") - -(let ((p (make-posix-process '("echo" "Hello" "World") nil nil))) - (start-process p)) \ No newline at end of file diff --git a/src/async-process.asd b/src/async-process.asd index eefc050..f929618 100644 --- a/src/async-process.asd +++ b/src/async-process.asd @@ -3,7 +3,6 @@ :author "cxxxr " :version "0.0.1" :license "MIT" - :depends-on ("cffi") :serial t :components ((:file "async-process_windows" :if-feature (:or :win32 :windows)) diff --git a/src/async-process.c b/src/async-process.c deleted file mode 100644 index 771bbe0..0000000 --- a/src/async-process.c +++ /dev/null @@ -1,374 +0,0 @@ -#include "async-process.h" - -int init_str(struct str *str) { - str->buf = malloc(sizeof(char) * 256); - if (str->buf == NULL) - return -1; - - str->len = 0; - str->cap = 256; - - return 0; -} - -void del_str(struct str *str) { - free(str->buf); -} - -static struct process* allocate_process(int fd_io, - int fd_er, - const char *pts_io_name, - const char *pts_er_name, - int pid) { - int stdout_ret = -1, stderr_ret = -1, both_ret = -1; - char *io_str = NULL, *er_str = NULL; - size_t io_strlen = strlen(pts_io_name) + 1; - size_t er_strlen = strlen(pts_er_name) + 1; - - struct process *process = malloc(sizeof(struct process)); - if (process == NULL) - return NULL; - - stdout_ret = init_str(&process->stdout); - if (stdout_ret == -1) - goto FAILED_MALLOC; - - stderr_ret = init_str(&process->stderr); - if (stderr_ret == -1) - goto FAILED_MALLOC; - - both_ret = init_str(&process->both); - if (both_ret == -1) - goto FAILED_MALLOC; - - io_str = malloc(io_strlen * sizeof(char)); - if (io_str == NULL) - goto FAILED_MALLOC; - - er_str = malloc(er_strlen * sizeof(char)); - if (er_str == NULL) - goto FAILED_MALLOC; - - memcpy(io_str, pts_io_name, io_strlen); - memcpy(er_str, pts_er_name, er_strlen); - - process->pts_io_name = io_str; - process->pts_er_name = er_str; - process->fd_io = fd_io; - process->fd_er = fd_er; - process->pid = pid; - - return process; - -FAILED_MALLOC: - if (process != NULL) free(process); - if (stdout_ret != -1) del_str(&process->stdout); - if (stderr_ret != -1) del_str(&process->stderr); - if (both_ret != -1) del_str(&process->both); - if (io_str != NULL) free(io_str); - if (er_str != NULL) free(er_str); - return NULL; -} - -void delete_process(struct process *process) { - kill(process->pid, 9); - close(process->fd_io); - close(process->fd_er); - del_str(&process->stdout); - del_str(&process->stderr); - del_str(&process->both); - free(process->pts_io_name); - free(process->pts_er_name); - free(process); -} - -void my_exit(int status) { - // exitを使うとatexitで動作に影響を与えられる、これが原因でプロセスを終了できなくなる事があるので使うのを避ける - // 例えばSDL2はat_exitを使っているせいか、lemのSDL2 frontendでasync_processが動作しなくなっていた - _exit(status); -} - -// opens a PTY and assigns master and slave file descriptors to fdm and fds -// respectively. Name will be malloced and it is the callers responsibility -// to free name. On success, return 0. On fail, returns -1. All references -// will also be either initialized or set -1/NULL appropriately. -// nonblock will set nonblock mode on the master PTY FD if nonblock == true -int open_pty(int *fdm, int *fds, char **name, bool nonblock) { - *fdm = -1; - *fds = -1; - *name = NULL; - - // gets a PTY, and initializes the attached slave PTS. grantpt and unlockpt - // are required before opening the slave device. - *fdm = posix_openpt(O_RDWR | O_NOCTTY); - if (*fdm == -1 || grantpt(*fdm) == -1 || unlockpt(*fdm) == -1) - goto FAILED_SETUP; - - // ptsname returns a string that must be copied, as it is overwritten - // on subsequent calls. - const char *tmp = ptsname(*fdm); - if (tmp == NULL) - goto FAILED_SETUP; - - size_t tmp_len = strlen(tmp) + 1; - *name = malloc(tmp_len * sizeof(char)); - if (*name == NULL) - goto FAILED_SETUP; - - memcpy(*name, tmp, tmp_len); - - *fds = open(*name, O_RDWR | O_NOCTTY); - if (*fds == -1) - goto FAILED_SETUP; - - // ensure both slave and master close after program finishes - fcntl(*fdm, F_SETFD, FD_CLOEXEC); - fcntl(*fds, F_SETFD, FD_CLOEXEC); - - // set master as non-blocking (for get_process_output functions) - if (nonblock) { - fcntl(*fdm, F_SETFL, O_NONBLOCK); - } - - // Set raw mode - struct termios tty; - tcgetattr(*fds, &tty); - cfmakeraw(&tty); - tcsetattr(*fds, TCSANOW, &tty); - - return 0; - -FAILED_SETUP: - if (*fdm != -1) close(*fdm); - if (*fds != -1) close(*fds); - if (*name != NULL) free(*name); - - *fdm = -1; - *fds = -1; - *name = NULL; - return -1; -} - -struct process* create_process(char *const command[], const char *path, bool nonblock) { - // Unix PTYs are bi-directional communication streams. Typically, a terminal will - // combine stdout and stderr and display them in the same output. We want to - // keep the outputs separate at this level so master_pty_er is created just to carry - // the stderr stream. - // - // There is a potential bug here, if the process tries to set terminal attributes (like - // with stty), these updates won't be propogated across both terminals. - - int master_pty_io, slave_pts_io, master_pty_er, slave_pts_er; - char *pts_io_name, *pts_er_name; - int ret; - - ret = open_pty(&master_pty_io, &slave_pts_io, &pts_io_name, nonblock); - if (ret == -1) - goto FAILED_SETUP; - - ret = open_pty(&master_pty_er, &slave_pts_er, &pts_er_name, nonblock); - if (ret == -1) - goto FAILED_SETUP; - - // START CHILD PROCESS AND RETURN ITS PID - pid_t pid = fork(); - - if (pid == -1) { - goto FAILED_SETUP; - } else if (pid != 0) { - close(slave_pts_io); - close(slave_pts_er); - // parent process, return process structure. - struct process *p = allocate_process(master_pty_io, master_pty_er, - pts_io_name, pts_er_name, pid); - - // allocate_process copies the strings it is passed, open_pty mallocs strings - // so we need to free them here before we exit. - free(pts_io_name); - free(pts_er_name); - return p; - } - - // VVV CHILD PROCESS VVV - setsid(); - - // we don't need these in the child process. - free(pts_io_name); - free(pts_er_name); - close(master_pty_io); - close(master_pty_er); - - dup2(slave_pts_io, STDIN_FILENO); - dup2(slave_pts_io, STDOUT_FILENO); - dup2(slave_pts_er, STDERR_FILENO); - - close(slave_pts_io); - close(slave_pts_er); - - if (path != NULL) chdir(path); - - // run command, the current fork process will switch to - // the command. - execvp(command[0], command); - - // if execution reaches here, there was a problem starting - // the program. execvp does not return on success. - int error_status = errno; - if (error_status == ENOENT) { - char str[128]; - sprintf(str, "%s: command not found", command[0]); - write(STDIN_FILENO, str, strlen(str)); - } else { - char *str = strerror(error_status); - write(STDIN_FILENO, str, strlen(str)); - } - my_exit(error_status); - - // ERROR HANDLING -FAILED_SETUP: - // we can assume at this point that any FD that is not -1 needs closed. - if (master_pty_io != -1) close(master_pty_io); - if (master_pty_er != -1) close(master_pty_er); - if (slave_pts_io != -1) close(slave_pts_io); - if (slave_pts_er != -1) close(slave_pts_er); - - // we can assume that any name pointer that is not NULL needs free. - if (pts_io_name != NULL) free(pts_io_name); - if (pts_er_name != NULL) free(pts_er_name); - - return NULL; -} - -int process_pid(struct process *process) { - return process->pid; -} - -// reads all data available in fd (should be non-blocking) into str, -// returns number of bytes read on success, -1 on error. -int str_read_fd(struct str *str, int fd) { - int total_read = 0; - while (true) { - // resize buffer if it is too small (include space for null terminator) - if (str->cap - str->len <= 1) { - char *new_ptr = realloc(str->buf, 2*str->cap); - if (new_ptr == NULL) - return -1; - str->buf = new_ptr; - str->cap *= 2; - } - - // read as much data from fd as possible. (read doesn't add '\0') - int n = read(fd, str->buf + str->len, str->cap - str->len); - - if (total_read == 0 && n == -1) { - return -1; // an error occured on first read - } else if (n <= 0) { - str->buf[str->len] = '\0'; // cap is always len+1 - return total_read; - } - - total_read += n; - str->len += n; - } - - return -1; // control flow shouldn't reach here. -} - -const char* _process_receive_fd(struct str *s, int fd, size_t *bytes) { - int n = str_read_fd(s, fd); - - if (n == -1 || s->len == 0) - return NULL; - - if (bytes != NULL) - *bytes = s->len; // length of str, not including '\0' - return s->buf; -} - -const char* process_receive_stdout(struct process *p, size_t *bytes) { - const char *r = _process_receive_fd(&p->stdout, p->fd_io, bytes); - p->stdout.len = 0; - return r; -} - -const char* process_receive_stderr(struct process *p, size_t *bytes) { - const char *r = _process_receive_fd(&p->stderr, p->fd_er, bytes); - p->stderr.len = 0; - return r; -} - -const char* process_receive_output(struct process *p, size_t *bytes) { - // these lengths include null terminators - size_t stdout_len = 0; - size_t stderr_len = 0; - - _process_receive_fd(&p->stdout, p->fd_io, &stdout_len); - _process_receive_fd(&p->stderr, p->fd_er, &stderr_len); - - if (p->both.cap < (stdout_len + stderr_len + 1)) { - char *new_ptr = realloc(p->both.buf, stdout_len + stderr_len); - if (new_ptr == NULL) - return NULL; - p->both.buf = new_ptr; - p->both.cap = stdout_len + stderr_len; - } - - memcpy(p->both.buf, p->stdout.buf, stdout_len); - memcpy(p->both.buf + stdout_len, p->stderr.buf, stderr_len); - p->both.buf[stdout_len + stderr_len] = '\0'; - - if (bytes != NULL) - *bytes = stdout_len + stderr_len; - - p->both.len = 0; - p->stdout.len = 0; - p->stderr.len = 0; - - return p->both.buf; -} - -ssize_t _process_write(struct process *process, const char *buf, size_t n, bool readp) { - ssize_t bytes_written = 0; - - while (bytes_written < n) { - ssize_t sent = write(process->fd_io, buf + bytes_written, n - bytes_written); - - if (readp) { - _process_receive_fd(&process->stdout, process->fd_io, NULL); - _process_receive_fd(&process->stderr, process->fd_er, NULL); - } - - if (bytes_written != 0 && sent == -1) { - return bytes_written; - } else if (sent == -1) { - return -1; - } - - bytes_written += sent; - } - - return bytes_written; -} - -ssize_t process_write(struct process *process, const char *buf, size_t n) { - return _process_write(process, buf, n, true); -} - -ssize_t process_write_string(struct process *process, const char *string) { - return _process_write(process, string, strlen(string), true); -} - -ssize_t process_write_noread(struct process *process, const char *buf, size_t n) { - return _process_write(process, buf, n, false); -} - -ssize_t process_write_string_noread(struct process *process, const char *string) { - return _process_write(process, string, strlen(string), false); -} - - - -int process_alive_p(struct process *process) -{ - return kill(process->pid, 0) == 0; -} diff --git a/src/async-process.h b/src/async-process.h deleted file mode 100644 index 8ca5974..0000000 --- a/src/async-process.h +++ /dev/null @@ -1,103 +0,0 @@ -#ifndef _ASYNC_PROCESS_H_ -#define _ASYNC_PROCESS_H_ - -#ifdef HAVE_CONFIG_H -# include "config.h" -#endif - -#define _GNU_SOURCE -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -struct str { - char* buf; - size_t len; - size_t cap; -}; - -int init_str(struct str *str); -void del_str(struct str *str); -int str_read_fd(struct str *str, int fd); - -struct process { - struct str stdout; - struct str stderr; - struct str both; - - int fd_io; - int fd_er; - char *pts_io_name; - char *pts_er_name; - pid_t pid; -}; - -struct process* create_process(char *const command[], const char *path, bool nonblock); -void delete_process(struct process *process); -int process_pid(struct process *process); - -/** Sends n bytes to process. - -returns the number of bytes written, or -1 indicating an error occurred. An -error will typically occur when the operating system cannot send all n bytes -because the PTY buffer is full. The process will have to read the buffer in -to make space for more data to be written. - -These functions read from the process STDOUT and STDERR buffers to keep -the process from being blocked. The results are buffered and will be returned -on the next call to a process_receive function. If these functions are used -in a separate thread from the process_receive functions, a race condition may -occur when this function is reading from STDOUT/STDERR (possibly realloc'ing) -while `process-receive_*` is reading from the same buffer. -*/ -ssize_t process_write(struct process *process, const char* buf, size_t n); -ssize_t process_write_string(struct process *process, const char *string); - -/** Sends n bytes to process. - -returns the number of bytes written, or -1 indicating an error occurred. An -error will typically occur when the operating system cannot send all n bytes -because the PTY buffer is full. The process will have to read the buffer in -to make space for more data to be written. - -Doesn't read devices STDOUT/STDERR file descriptors. If `process_receive* -functions are not called regularly, the internal PTY buffers may fill and -prevent the attached process from continuing to run. `process_write` and -`process_write_string` prevent this from happening, but their usage requires -other considerations. -*/ -ssize_t process_write_noread(struct process *process, const char* buf, size_t n); -ssize_t process_write_string_noread(struct process *process, const char *string); - -/** Return Process STDOUT. -Returns pointer to a buffer containing data returned by process STDOUT buffer. -this buffer will be overwritten by subsequent calls to this function; if -this output is meant to be kept, it should be copied out. -*/ -const char* process_receive_stdout(struct process *process, size_t *bytes); - -/** Return Process STDERR. -Returns pointer to a buffer containing data returned by process STDERR buffer. -this buffer will be overwritten by subsequent calls to this function; if -this output is meant to be kept, it should be copied out. -*/ -const char* process_receive_stderr(struct process *process, size_t *bytes); - -/** Receive Process STDOUT and STDERR (one after another). -Returns pointer to a buffer containing data returned by process STDERR and -STDOUT buffer. this buffer will be overwritten by subsequent calls to this - function; if this output is meant to be kept, it should be copied out. -*/ -const char* process_receive_output(struct process *process, size_t *bytes); - -int process_alive_p(struct process *process); - -#endif diff --git a/src/async-process.lisp b/src/async-process.lisp deleted file mode 100644 index 42e0cb8..0000000 --- a/src/async-process.lisp +++ /dev/null @@ -1,154 +0,0 @@ -(defpackage :async-process - (:use :cl) - (:export - :delete-process - :process-send-input - :process-receive-output - :process-alive-p - :create-process - :cffi-test)) -(in-package :async-process) - -(eval-when (:compile-toplevel :load-toplevel :execute) - (defun system (cmd) - (ignore-errors (string-right-trim '(#\Newline) (uiop:run-program cmd :output :string)))) - (defun muslp () - (ignore-errors - (not (zerop (length (uiop:run-program - "ldd /bin/ls |grep musl" - :ignore-error-status t - :output :string))))))) - -(format t "~&async process is at: ~A" - (asdf:system-relative-pathname "async-process" ".")) - -(pushnew (asdf:system-relative-pathname - :async-process - (format nil "../.libs/~A/" - (cond - ;; Windows - ((uiop/os:featurep '(:and :windows :x86-64)) - "x86_64/windows") - ((uiop/os:featurep :windows) - "x86/windows") - ;; macOS (Darwin) - ((uiop/os:featurep :os-macosx) - (format nil "~A/darwin" - (uiop:run-program '("uname" "-m") :output '(:string :stripped t)))) - ;; Linux / Generic Unix - ((uiop/os:featurep :unix) - (format nil "~A/~A" - (uiop:run-program '("uname" "-m") :output '(:string :stripped t)) - (let ((os (uiop:run-program '("uname") :output '(:string :stripped t)))) - (cond ((and (equal os "Linux") - (ignore-errors (funcall (read-from-string "muslp"))))n - "Linux-musl") - (t os)))))))) - cffi:*foreign-library-directories* - :test #'uiop:pathname-equal) - -(pushnew (asdf:system-relative-pathname "async-process" "../.libs/") cffi:*foreign-library-directories*) - -;; this binds the library file path to the symbol async-process for use with -;; `use-foreign-library` -(format t "~&cffi foreign library status: ~A" - (cffi:define-foreign-library async-process - (:darwin "libasyncprocess.dylib") - (:unix "libasyncprocess.so") - (:windows "libasyncprocess.dll"))) - -;; This currently fails because qlot does not copy the .libs folder over. -;;(cffi:use-foreign-library async-process) - -(defclass process () - ((process :reader process-process :initarg :process) - (encode :accessor process-encode :initarg :encode))) - -(cffi:defcfun ("create_process" %create-process) :pointer - (command :pointer) - (path :string) - (noblock :bool)) - -(cffi:defcfun ("delete_process" %delete-process) :void - (process :pointer)) - -(cffi:defcfun ("process_pid" %process-pid) :int - (process :pointer)) - -(cffi:defcfun ("process_write" %process-write) :ssize - (process :pointer) - (string :string) - (n :size)) - -(cffi:defcfun ("process_write_string" %process-write-string) :ssize - (process :pointer) - (string :string)) - -(cffi:defcfun ("process_receive_stdout" %process-receive-stdout) :pointer - (process :pointer) - (bytes :pointer)) - -(cffi:defcfun ("process_receive_stderr" %process-receive-stderr) :pointer - (process :pointer) - (bytes :pointer)) - -(cffi:defcfun ("process_receive_output" %process-receive-output) :pointer - (process :pointer) - (bytes :pointer)) - -(cffi:defcfun ("process_alive_p" %process-alive-p) :boolean - (process :pointer)) - -(cffi:defcfun "cffi_test" :string) - -(defun create-process (command &key nonblock (encode cffi:*default-foreign-encoding*) directory) - (when (and directory (not (uiop:directory-exists-p directory))) - (error "Directory ~S does not exist" directory)) - (let* ((command (uiop:ensure-list command)) - (length (length command))) - (cffi:with-foreign-object (argv :string (1+ length)) - (loop :for i :from 0 - :for c :in command - :do (setf (cffi:mem-aref argv :string i) c)) - (setf (cffi:mem-aref argv :string length) (cffi:null-pointer)) - (let ((p (%create-process argv - (if directory - (namestring directory) - (cffi:null-pointer)) - nonblock))) - (if (cffi:null-pointer-p p) - (error "create-process failed: ~S" command) - (make-instance 'process :process p :encode encode)))))) - -(defun delete-process (process) - (%delete-process (process-process process))) - -(defun process-pid (process) - (%process-pid (process-process process))) - -(defun process-send-input (process string) - (let ((cffi:*default-foreign-encoding* (process-encode process))) - (%process-write-string (process-process process) string))) - -(defun process-receive-output (process &optional (source :both)) - "`source` can be either `:stdout`, `:stderr`, or `:both`. It specifies the stream -to read from." - (declare (optimize (debug 3))) - (flet ((call-cfun (read-func) - "helper function to call one of the three cffi functions for receiving output." - (cffi:with-foreign-pointer (bytes 8) - (let ((cffi:*default-foreign-encoding* (process-encode process)) - (output (funcall read-func - (process-process process) - bytes))) - (cffi:foreign-string-to-lisp - output - :count (cffi:mem-ref bytes :size)))))) - - (case source - (:stdout (call-cfun '%process-receive-stdout)) - (:stderr (call-cfun '%process-receive-stderr)) - (:both (call-cfun '%process-receive-output))))) - -(defun process-alive-p (process) - (%process-alive-p (process-process process))) diff --git a/src/libc-symbols-grovel.lisp b/src/libc-symbols-grovel.lisp deleted file mode 100644 index 6efe2e4..0000000 --- a/src/libc-symbols-grovel.lisp +++ /dev/null @@ -1,29 +0,0 @@ -(in-package async-process) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(include "fcntl.h") -(constant (+o-rdwr+ "O_RDWR")) -(constant (+o-noctty+ "O_NOCTTY")) -(constant (+o_nonblock+ "O_NONBLOCK")) - -(constant (+f-setfd+ "F_SETFD")) -(constant (+f-setfl+ "F_SETFL")) - -(constant (+fd-cloexec+ "FD_CLOEXEC")) - -(constant (+tcsanow+ "TCSANOW")) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(include "unistd.h") -(constant (+stdin-fileno+ "STDIN_FILENO")) -(constant (+stdout-fileno+ "STDOUT_FILENO")) -(constant (+stderr-fileno+ "STDERR_FILENO")) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(include "errno.h") -(cvar ("errno" *errno*) :int) -(constant (+enoent+ "ENOENT")) - -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(include "termios.h") -(cstruct termios "struct termios") diff --git a/src/test.c b/src/test.c deleted file mode 100644 index 37e6ecb..0000000 --- a/src/test.c +++ /dev/null @@ -1,58 +0,0 @@ -#include "async-process.h" - -int main() { - char *cmd[] = {"tee", "ima-cool-file", NULL}; - struct process *p = create_process(cmd, NULL); - - struct str s; - init_str(&s); - - #define TEST_INPUT_SIZE 500000 - char test_input[TEST_INPUT_SIZE]; - - for (size_t i = 0; i < TEST_INPUT_SIZE; i+=10) { - memcpy(test_input+i, "123456789\n", 10); - } - - size_t n = 0; - while (n != TEST_INPUT_SIZE) { - ssize_t bytes = process_write(p, test_input+n, TEST_INPUT_SIZE-n); - if (bytes > 0) { - printf("wrote %d/%d bytes...\n", n, TEST_INPUT_SIZE); - n += bytes; - } else { - printf("%s: %s\n", strerrorname_np(errno), strerror(errno)); - } - - } - - printf("I just attempted to write %d.\nI wrote %d bytes.\n", TEST_INPUT_SIZE, n); - - while (true) { - int n = str_read_fd(&s, STDIN_FILENO); - if (n > 0) { - process_write(p, s.buf, n); - s.len = 0; - if (strcmp(s.buf, "exit\n") == 0) - break; - } - - const char *out = NULL; - const char *err = NULL; - - out = process_receive_stdout(p); - err = process_receive_stderr(p); - - if (out != NULL) { - printf("%s", out); - } - - if (err != NULL) { - printf("\033[31m%s\033[0m", err); - } - } - - delete_process(p); - del_str(&s); - return 0; -} diff --git a/src/test.lisp b/src/test.lisp deleted file mode 100644 index 5b02724..0000000 --- a/src/test.lisp +++ /dev/null @@ -1,35 +0,0 @@ -asdf:*central-registry* -(ql:quickload "alexandria") -(ql:quickload "babel") - -(setf asdf:*central-registry* (list #P"/home/ethan/Documents/async-process/src/")) -(asdf:load-asd #P"/home/ethan/Documents/async-process/src/async-process.asd") - -(asdf:load-system "async-process") - -(defvar *proc* nil) -(setf *proc* (async-process:create-process '("tee" "/home/ethan/test.log") - :nonblock t)) - -(format t "~&~a" - (with-output-to-string (s) - (async-process:process-send-input *proc* (format nil "ima bot~%")) - (sleep 0.1) - (format s "~A" (async-process:process-receive-output *proc*)))) - -(async-process:process-send-input *proc* (format nil "bop~%")) -(format t (async-process:process-receive-output *proc* :both)) - -(format t "~&~S" (async-process:process-receive-output *proc* :both)) - -(defun cffi-null-string-test () - (format t "~&~S" - (cffi:with-pointer-to-vector-data - (p (make-array 10 - :element-type '(unsigned-byte 8) - :initial-element 0)) - (cffi:foreign-string-to-lisp p)))) - -;;; test which cffi lib is currently loaded. -(format t "~&---~%~{~A~%~}" (cffi:list-foreign-libraries)) -(format t "~A" (cffi:foreign-library-pathname 'async-process::async-process)) From d6e9e5aa53b06bcd2d81cdf73f820d083a5e4fe1 Mon Sep 17 00:00:00 2001 From: ethanxxxl Date: Fri, 17 Apr 2026 23:07:06 -0400 Subject: [PATCH 15/17] simplified project structure, removed c-lib stuff --- GNUmakefile | 63 ------------------ Makefile.am | 8 --- src/async-process.asd => async-process.asd | 6 +- ...nc-process-uiop.lisp => async-process.lisp | 0 ...windows.lisp => async-process_windows.lisp | 0 configure.ac | 64 ------------------- 6 files changed, 3 insertions(+), 138 deletions(-) delete mode 100644 GNUmakefile delete mode 100644 Makefile.am rename src/async-process.asd => async-process.asd (67%) rename src/async-process-uiop.lisp => async-process.lisp (100%) rename src/async-process_windows.lisp => async-process_windows.lisp (100%) delete mode 100644 configure.ac diff --git a/GNUmakefile b/GNUmakefile deleted file mode 100644 index 4323451..0000000 --- a/GNUmakefile +++ /dev/null @@ -1,63 +0,0 @@ -PREFIX ?= /usr/local - -GENERATED_MAKEFILE := $(wildcard Makefile) - -.PHONY: distclean -distclean: - @if [ -f Makefile ]; then \ - echo "Running make distclean..."; \ - $(MAKE) -f Makefile distclean 2>/dev/null || true; \ - fi - @echo "Removing autotools generated files..." - @rm -rf Makefile Makefile.in configure config.* libtool aclocal.m4 stamp-h1 autom4te.cache - @rm -rf compile config.guess config.sub depcomp install-sh ltmain.sh missing - @rm -rf .libs .deps src/.libs src/.deps - @rm -f *.lo *.la src/*.lo src/*.la - @echo "All generated files removed. Run 'make' to rebuild." - -# If Makefile exists, delegate to it -ifneq ($(GENERATED_MAKEFILE),) - -.DEFAULT_GOAL := all - -%: - $(MAKE) -f Makefile $@ - -else - -# No Makefile - need to generate build system -.DEFAULT_GOAL := all - -configure: configure.ac Makefile.am - @echo "Generating build system..." - @which glibtoolize > /dev/null 2>&1 && glibtoolize --copy --force --quiet || libtoolize --copy --force --quiet - @aclocal - @autoheader - @automake --add-missing --copy --foreign - @autoconf - @echo "" - -Makefile: configure - @echo "Running configure..." - @./configure --prefix=$(PREFIX) - @echo "" - -.PHONY: all -all: Makefile - @$(MAKE) -f Makefile all - @echo "" - @echo "Build complete. Install with: make install" - @echo "" - -.PHONY: build -build: all - -.PHONY: install -install: Makefile - @$(MAKE) -f Makefile install - -.PHONY: clean -clean: Makefile - @$(MAKE) -f Makefile clean - -endif diff --git a/Makefile.am b/Makefile.am deleted file mode 100644 index a4f6213..0000000 --- a/Makefile.am +++ /dev/null @@ -1,8 +0,0 @@ -lib_LTLIBRARIES = libasyncprocess.la -libasyncprocess_la_LDFLAGS = -version-info @LT_VERSION_INFO@ -no-undefined -libasyncprocess_la_SOURCES = src/async-process.c - -include_HEADERS = src/async-process.h - -install-exec-hook: - rm -f $(DESTDIR)$(libdir)/libasyncprocess.la diff --git a/src/async-process.asd b/async-process.asd similarity index 67% rename from src/async-process.asd rename to async-process.asd index f929618..f83d852 100644 --- a/src/async-process.asd +++ b/async-process.asd @@ -1,10 +1,10 @@ (defsystem "async-process" :description "asynchronous process execution for common lisp" - :author "cxxxr " - :version "0.0.1" + :author "cxxxr , Ethan Smith Date: Sat, 18 Apr 2026 00:16:57 -0400 Subject: [PATCH 16/17] updated readme, added option to read error output --- README.md | 116 ++++++--------------------------------------- async-process.asd | 1 + async-process.lisp | 7 ++- 3 files changed, 21 insertions(+), 103 deletions(-) diff --git a/README.md b/README.md index 7231398..b44817f 100644 --- a/README.md +++ b/README.md @@ -1,114 +1,28 @@ # async-process -A Common Lisp library for creating and managing asynchronous processes with PTY support. +A Common Lisp library for creating and managing asynchronous processes ## Platform Support -- **Linux**: Full support via C library using PTY -- **BSD**: Full support via C library using PTY -- **macOS**: Full support via C library using PTY -- **Windows**: Full support via pure Lisp CFFI implementation (no C compilation required) - -## Installation - -The build system is GNU Autotools, with a somewhat non-standard -setup. - -The default goal `all` in `GNUmakefile` can perform the following -sequence in one go: -1. Run `autoconf` using `configure.ac` and `Makefile.am` as inputs. -2. Run `configure` (generated in step 1) to produce a `Makefile`. -3. Build and compile using the generated `Makefile`. - -### Unix-like Systems (Linux, FreeBSD, macOS) -using GNU make, `gmake` on Freebsd and macOS. - -```bash -git clone https://github.com/lem-project/async-process.git -cd async-process -make -sudo make install -``` - -The library installs to `/usr/local` by default. To install elsewhere: - -To install to a different destination, the `make` command should -be substituted with another similar to the examples below, passing -an explicitly-set `PREFIX` environment variable. - -```bash -PREFIX=/your/custom/path make -PREFIX=/usr make -PREFIX=$HOME/.local make -``` - -Alternatively, you can run the Autotools toolchain sequence -as follows, with a slightly different method of setting the -destination prefix: - -```bash -autoreconf -i -./configure --prefix=/your/custom/path -make -make install -``` - -#### Configuration options -Build as a static library as follows - -```bash -make -./configure --enable-static -make all -sudo make install -``` - -### Windows - -On Windows, no C compilation is required. The library uses a pure Lisp implementation via CFFI: - -```bash -git clone https://github.com/lem-project/async-process.git -cd async-process -``` - -Then simply load the library in your Lisp environment: - -```lisp -(ql:quickload :async-process) -``` +- **Linux**: Full support via UIOP +- **BSD**: Full support via UIOP +- **macOS**: Full support via UIOP +- **Windows**: Full support via CFFI implementation The ASDF system will automatically load the Windows-specific implementation (`src/async-process_windows.lisp`) when on Windows platforms. +The the UIOP implementation may also work on windows. Once it is tested, it may be +switched over. + ## Usage -``` -CL-USER> (ql:quickload :async-process) -To load "async-process": - Load 1 ASDF system: - async-process -; Loading "async-process" -.................................................. -[package async-process]. -(:ASYNC-PROCESS) -CL-USER> (in-package async-process) -# -ASYNC-PROCESS> (create-process "python") -#.(SB-SYS:INT-SAP #X7FFFEC002830) -ASYNC-PROCESS> (defparameter p *) -#.(SB-SYS:INT-SAP #X7FFFEC002830) -ASYNC-PROCESS> (process-receive-output p) -"Python 2.7.13 (default, Nov 24 2017, 17:33:09) -[GCC 6.3.0 20170516] on linux2 -Type \"help\", \"copyright\", \"credits\" or \"license\" for more information. ->>> " -ASYNC-PROCESS> (process-send-input p "1+1 -") -; No value -ASYNC-PROCESS> (process-receive-output p) -"1+1 -2 ->>> " +```lisp +(ql:quickload "async-process") +(in-package async-process) ; => # +(defparameter p (create-process '("tee"))) ; => P +(process-send-input p "hello world") ; => NIL +(process-receive-output p) ; => "hello world" +(delete-process p) ; => T ``` ## LICENSE diff --git a/async-process.asd b/async-process.asd index f83d852..b3e2a08 100644 --- a/async-process.asd +++ b/async-process.asd @@ -3,6 +3,7 @@ :author "cxxxr , Ethan Smith Date: Fri, 1 May 2026 12:26:42 -0400 Subject: [PATCH 17/17] added with-process macro and cleaned up testing code. --- async-process-test.lisp | 54 +++++++++++++++++++++++++ async-process.asd | 2 +- async-process.lisp | 88 ++++++++++++++++++++--------------------- 3 files changed, 97 insertions(+), 47 deletions(-) create mode 100644 async-process-test.lisp diff --git a/async-process-test.lisp b/async-process-test.lisp new file mode 100644 index 0000000..fb5437e --- /dev/null +++ b/async-process-test.lisp @@ -0,0 +1,54 @@ +(ql:quickload 'rove) +(defpackage async-process/test + (:use :cl + :rove + :async-process)) + +(in-package async-process/test) + +(deftest all-tests + (testing "Process setup/cleanup" + (ok (let ((procs (copy-list async-process::*active-processes*)) + (p)) + (setf p (create-process '("true") :nonblock nil)) + (delete-process p) + (equal procs async-process::*active-processes*))) + + (ok (let ((procs (copy-list async-process::*active-processes*))) + (with-process () ('("true"))) + (equal procs async-process::*active-processes*)))) + + (testing "Process input/output" + (ok (outputs (let ((proc (create-process '("echo" "hello" "world") :nonblock nil))) + (princ (process-receive-output proc)) + (delete-process proc)) + (format nil "hello world~%"))) + + (ok (outputs (with-process (:name p) ('("echo" "hello" "world")) + (format t "~A" (process-receive-output p))) + (format nil "hello world~%"))) + + (ok (outputs (with-process (:output s) ('("echo" "hello" "world") :nonblock nil) + (princ (read-line s))) + (format nil "hello world"))) + + (ok (outputs (with-process (:output s) ('("echo" "hello" "world")) + (format t "~A" (read-line s))) + (format nil "hello world"))) + + (ok (outputs (with-process (:name p) ('("echo" "hello" "world") :nonblock t) + (format t "~A" (process-receive-output p))) + (format nil ""))) + + (ok (outputs (with-process (:name p) ('("tee") :nonblock nil) + (process-send-input p "hello world") + (format t "~A" (process-receive-output p))) + + (format nil "hello world"))) + + (ok (outputs (with-process (:input in :output out) ('("tee") :nonblock nil) + (format in "hello world~%") + (finish-output in) + (format t "~A" (read-line out))) + + (format nil "hello world"))))) diff --git a/async-process.asd b/async-process.asd index b3e2a08..5c4412f 100644 --- a/async-process.asd +++ b/async-process.asd @@ -3,7 +3,7 @@ :author "cxxxr , Ethan Smith