From 9c73da85d667ca682d8a77641f39368d868658f1 Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Tue, 20 Aug 2019 00:04:31 -0300 Subject: [PATCH 1/9] Start porting ob-julia.el for org 9.2.5 - Stop relying on ess to set current buffer - Remove some extra newlines --- ob-julia.el | 163 +++++++++++++++++++++++++++------------------------- 1 file changed, 85 insertions(+), 78 deletions(-) diff --git a/ob-julia.el b/ob-julia.el index e88f112..405402d 100644 --- a/ob-julia.el +++ b/ob-julia.el @@ -6,16 +6,16 @@ ;; Org-Babel support for evaluating julia code ;;; Code: +(require 'cl-lib) (require 'ob) -(eval-when-compile (require 'cl)) (declare-function orgtbl-to-csv "org-table" (table params)) (declare-function julia "ext:ess-julia" (&optional start-args)) (declare-function inferior-ess-send-input "ext:ess-inf" ()) (declare-function ess-make-buffer-current "ext:ess-inf" ()) (declare-function ess-eval-buffer "ext:ess-inf" (vis)) -(declare-function org-number-sequence "org-compat" (from &optional to inc)) -(declare-function org-remove-if-not "org" (predicate seq)) +(declare-function ess-wait-for-process "ext:ess-inf" + (&optional proc sec-prompt wait force-redisplay)) (defconst org-babel-header-args:julia '((width . :any) @@ -30,54 +30,66 @@ (defvar org-babel-default-header-args:julia '()) -(defcustom org-babel-julia-command inferior-julia-program-name +(defvar inferior-julia-program "julia") + +(defcustom org-babel-julia-command inferior-julia-program "Name of command to use for executing julia code." :group 'org-babel :version "24.3" :type 'string) +(defvar ess-current-process-name) ; dynamically scoped (defvar ess-local-process-name) ; dynamically scoped (defun org-babel-edit-prep:julia (info) - (let ((session (cdr (assoc :session (nth 2 info))))) - (when (and session (string-match "^\\*\\(.+?\\)\\*$" session)) - (save-match-data (org-babel-julia-initiate-session session nil))))) + (let ((session (cdr (assq :session (nth 2 info))))) + (when (and session + (string-prefix-p "*" session) + (string-suffix-p "*" session)) + (org-babel-julia-initiate-session session nil)))) -(defun org-babel-expand-body:julia (body params &optional graphics-file) +(defun org-babel-expand-body:julia (body params &optional _graphics-file) "Expand BODY according to PARAMS, return the expanded body." - (let ((graphics-file - (or graphics-file (org-babel-julia-graphical-output-file params)))) - (mapconcat - #'identity - ((lambda (inside) - (if graphics-file - inside - inside) - ) - (append (org-babel-variable-assignments:julia params) - (list body))) "\n"))) + (mapconcat 'identity + (append + (when (cdr (assq :prologue params)) + (list (cdr (assq :prologue params)))) + (org-babel-variable-assignments:julia params) + (list body) + (when (cdr (assq :epilogue params)) + (list (cdr (assq :epilogue params))))) + "\n")) (defun org-babel-execute:julia (body params) "Execute a block of julia code. This function is called by `org-babel-execute-src-block'." (save-excursion - (let* ((result-params (cdr (assoc :result-params params))) - (result-type (cdr (assoc :result-type params))) + (let* ((result-params (cdr (assq :result-params params))) + (result-type (cdr (assq :result-type params))) (session (org-babel-julia-initiate-session - (cdr (assoc :session params)) params)) - (colnames-p (cdr (assoc :colnames params))) - (rownames-p (cdr (assoc :rownames params))) - (graphics-file (org-babel-julia-graphical-output-file params)) + (cdr (assq :session params)) params)) + (graphics-file (and (member "graphics" (assq :result-params params)) + (org-babel-graphical-output-file params))) + (colnames-p (unless graphics-file (cdr (assq :colnames params)))) + (rownames-p (unless graphics-file (cdr (assq :rownames params)))) (full-body (org-babel-expand-body:julia body params graphics-file)) (result (org-babel-julia-evaluate session full-body result-type result-params (or (equal "yes" colnames-p) (org-babel-pick-name - (cdr (assoc :colname-names params)) colnames-p)) + (cdr (assq :colname-names params)) colnames-p)) (or (equal "yes" rownames-p) (org-babel-pick-name - (cdr (assoc :rowname-names params)) rownames-p))))) - (if graphics-file nil result)))) + (cdr (assq :rowname-names params)) rownames-p))))) + (if graphics-file nil (if (and result (sequencep result)) + (org-babel-normalize-newline result) + result))))) + +(defun org-babel-normalize-newline (result) + (replace-regexp-in-string + "\\(\n\r?\\)\\{2,\\}" + "" + result)) (defun org-babel-prep-session:julia (session params) "Prepare SESSION according to the header arguments specified in PARAMS." @@ -102,23 +114,21 @@ This function is called by `org-babel-execute-src-block'." (defun org-babel-variable-assignments:julia (params) "Return list of julia statements assigning the block's variables." - (let ((vars (if (fboundp 'org-babel-get-header) - (mapcar #'cdr (org-babel-get-header params :var)) - (mapcar #'cdr (org-babel--get-vars params))))) + (let ((vars (org-babel--get-vars params))) (mapcar (lambda (pair) (org-babel-julia-assign-elisp (car pair) (cdr pair) - (equal "yes" (cdr (assoc :colnames params))) - (equal "yes" (cdr (assoc :rownames params))))) + (equal "yes" (cdr (assq :colnames params))) + (equal "yes" (cdr (assq :rownames params))))) (mapcar (lambda (i) (cons (car (nth i vars)) (org-babel-reassemble-table (cdr (nth i vars)) - (cdr (nth i (cdr (assoc :colname-names params)))) - (cdr (nth i (cdr (assoc :rowname-names params))))))) - (org-number-sequence 0 (1- (length vars))))))) + (cdr (nth i (cdr (assq :colname-names params)))) + (cdr (nth i (cdr (assq :rowname-names params))))))) + (number-sequence 0 (1- (length vars))))))) (defun org-babel-julia-quote-csv-field (s) "Quote field S for export to julia." @@ -129,40 +139,37 @@ This function is called by `org-babel-execute-src-block'." (defun org-babel-julia-assign-elisp (name value colnames-p rownames-p) "Construct julia code assigning the elisp VALUE to a variable named NAME." (if (listp value) - (let ((max (apply #'max (mapcar #'length (org-remove-if-not - #'sequencep value)))) - (min (apply #'min (mapcar #'length (org-remove-if-not - #'sequencep value)))) - (transition-file (org-babel-temp-file "julia-import-"))) - ;; ensure VALUE has an orgtbl structure (depth of at least 2) + (let* ((lengths (mapcar 'length (cl-remove-if-not 'sequencep value))) + (max (if lengths (apply 'max lengths) 0)) + (min (if lengths (apply 'min lengths) 0))) + ;; Ensure VALUE has an orgtbl structure (depth of at least 2). (unless (listp (car value)) (setq value (list value))) - (with-temp-file transition-file - (insert - (orgtbl-to-csv value '(:fmt org-babel-julia-quote-csv-field)) - "\n")) - (let ((file (org-babel-process-file-name transition-file 'noquote)) - (header (if (or (eq (nth 1 value) 'hline) colnames-p) - "TRUE" "FALSE")) - (row-names (if rownames-p "1" "NULL"))) - (if (= max min) - (format "%s = readcsv(\"%s\")" name file) - (format "%s = readcsv(\"%s\")" - name file)))) + (let ((file (orgtbl-to-tsv value '(:fmt org-babel-julia-quote-tsv-field))) + (header (if (or (eq (nth 1 value) 'hline) colnames-p) + "TRUE" "FALSE")) + (row-names (if rownames-p "1" "NULL"))) + (if (= max min) + (format "%s = readdlm(\"%s\")" name file) + (format "%s = readdlm(\"%s\")" + name file)))) (format "%s = %s" name (org-babel-julia-quote-csv-field value)))) (defvar ess-ask-for-ess-directory) ; dynamically scoped - (defun org-babel-julia-initiate-session (session params) "If there is not a current julia process then create one." (unless (string= session "none") - (let ((session (or session "*julia*")) + (let ((session (or session "*Julia*")) (ess-ask-for-ess-directory - (and (and (boundp 'ess-ask-for-ess-directory) ess-ask-for-ess-directory) - (not (cdr (assoc :dir params)))))) + (and (boundp 'ess-ask-for-ess-directory) + ess-ask-for-ess-directory + (not (cdr (assq :dir params)))))) (if (org-babel-comint-buffer-livep session) session (save-window-excursion - (require 'ess) (julia) + (when (get-buffer session) + ;; Session buffer exists, but with dead process + (set-buffer session)) + (require 'ess) (set-buffer (julia)) (rename-buffer (if (bufferp session) (buffer-name session) @@ -171,13 +178,13 @@ This function is called by `org-babel-execute-src-block'." (buffer-name)))) (current-buffer)))))) -(defun org-babel-julia-associate-session (session) - "Associate julia code buffer with a julia session. -Make SESSION be the inferior ESS process associated with the -current code buffer." - (setq ess-local-process-name - (process-name (get-buffer-process session))) - (ess-make-buffer-current)) +; (defun org-babel-julia-associate-session (session) +; "Associate julia code buffer with a julia session. +; Make SESSION be the inferior ESS process associated with the +; current code buffer." +; (setq ess-local-process-name +; (process-name (get-buffer-process session))) +; (ess-make-buffer-current)) (defun org-babel-julia-graphical-output-file (params) "Name of file to which julia should send graphical output." @@ -187,11 +194,10 @@ current code buffer." (defvar org-babel-julia-eoe-indicator "print(\"org_babel_julia_eoe\")") (defvar org-babel-julia-eoe-output "org_babel_julia_eoe") -(defvar org-babel-julia-write-object-command "writecsv(\"%s\",%s)") - -;; The following was a very complicated write object command -;; The replacement needs to add error catching -;(defvar org-babel-julia-write-object-command "{function(object,transfer.file){object;invisible(if(inherits(try({tfile<-tempfile();write.table(object,file=tfile,sep=\"\\t\",na=\"nil\",row.names=%s,col.names=%s,quote=FALSE);file.rename(tfile,transfer.file)},silent=TRUE),\"try-error\")){if(!file.exists(transfer.file))file.create(transfer.file)})}}(object=%s,transfer.file=\"%s\")") +(defvar org-babel-julia-write-object-command "begin + using DelimitedFiles + writedlm(\"%s\", \",\", [ %s ]) +end") (defun org-babel-julia-evaluate (session body result-type result-params column-names-p row-names-p) @@ -208,13 +214,13 @@ current code buffer." If RESULT-TYPE equals 'output then return standard output as a string. If RESULT-TYPE equals 'value then return the value of the last statement in BODY, as elisp." - (case result-type + (cl-case result-type (value (let ((tmp-file (org-babel-temp-file "julia-"))) (org-babel-eval org-babel-julia-command (format org-babel-julia-write-object-command (org-babel-process-file-name tmp-file 'noquote) - (format "begin\n%s\nend" body))) + (format "(function () %s end)()" body))) (org-babel-julia-process-value-result (org-babel-result-cond result-params (with-temp-buffer @@ -230,7 +236,7 @@ last statement in BODY, as elisp." If RESULT-TYPE equals 'output then return standard output as a string. If RESULT-TYPE equals 'value then return the value of the last statement in BODY, as elisp." - (case result-type + (cl-case result-type (value (with-temp-buffer (insert (org-babel-chomp body)) @@ -252,7 +258,7 @@ last statement in BODY, as elisp." column-names-p))) (output (mapconcat - #'org-babel-chomp + 'org-babel-chomp (butlast (delq nil (mapcar @@ -260,14 +266,15 @@ last statement in BODY, as elisp." (mapcar (lambda (line) ;; cleanup extra prompts left in output (if (string-match - "^\\([ ]*[>+\\.][ ]?\\)+\\([[0-9]+\\|[ ]\\)" line) + "^\\([>+.]\\([ ][>.+]\\)*[ ]\\)" + (car (split-string line "\n"))) (substring line (match-end 1)) line)) (org-babel-comint-with-output (session org-babel-julia-eoe-output) - (insert (mapconcat #'org-babel-chomp + (insert (mapconcat 'org-babel-chomp (list body org-babel-julia-eoe-indicator) "\n")) - (inferior-ess-send-input)))))) "\n")))) + (inferior-ess-send-input)))))) "\n")))) (defun org-babel-julia-process-value-result (result column-names-p) "julia-specific processing of return value. From 81a5cbdf0ae0a956e0a1b90b2860933a0deaea32 Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Tue, 20 Aug 2019 16:33:30 -0300 Subject: [PATCH 2/9] Start to fix :results value execution --- ob-julia.el | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/ob-julia.el b/ob-julia.el index 405402d..4dcbf4f 100644 --- a/ob-julia.el +++ b/ob-julia.el @@ -81,14 +81,14 @@ This function is called by `org-babel-execute-src-block'." (or (equal "yes" rownames-p) (org-babel-pick-name (cdr (assq :rowname-names params)) rownames-p))))) - (if graphics-file nil (if (and result (sequencep result)) - (org-babel-normalize-newline result) - result))))) + (if graphics-file nil (if (and result (sequencep result)) + (org-babel-normalize-newline result) + result))))) (defun org-babel-normalize-newline (result) (replace-regexp-in-string "\\(\n\r?\\)\\{2,\\}" - "" + "\n" result)) (defun org-babel-prep-session:julia (session params) @@ -149,8 +149,14 @@ This function is called by `org-babel-execute-src-block'." "TRUE" "FALSE")) (row-names (if rownames-p "1" "NULL"))) (if (= max min) - (format "%s = readdlm(\"%s\")" name file) - (format "%s = readdlm(\"%s\")" + (format "%s = begin + using CSV + CSV.read(\"%s\") +end" name file) + (format "%s = begin + using CSV + CSV.read(\"%s\") +end" name file)))) (format "%s = %s" name (org-babel-julia-quote-csv-field value)))) @@ -195,8 +201,10 @@ This function is called by `org-babel-execute-src-block'." (defvar org-babel-julia-eoe-output "org_babel_julia_eoe") (defvar org-babel-julia-write-object-command "begin - using DelimitedFiles - writedlm(\"%s\", \",\", [ %s ]) + local p_ans = [ %s ] + using CSV + CSV.write(\"%s\", p_ans) + p_ans end") (defun org-babel-julia-evaluate @@ -219,8 +227,9 @@ last statement in BODY, as elisp." (let ((tmp-file (org-babel-temp-file "julia-"))) (org-babel-eval org-babel-julia-command (format org-babel-julia-write-object-command + (format "(function () %s end)()" body) (org-babel-process-file-name tmp-file 'noquote) - (format "(function () %s end)()" body))) + )) (org-babel-julia-process-value-result (org-babel-result-cond result-params (with-temp-buffer @@ -248,7 +257,9 @@ last statement in BODY, as elisp." (org-babel-comint-eval-invisibly-and-wait-for-file session tmp-file (format org-babel-julia-write-object-command - (org-babel-process-file-name tmp-file 'noquote) "ans")) + "ans" + (org-babel-process-file-name tmp-file 'noquote) + )) (org-babel-julia-process-value-result (org-babel-result-cond result-params (with-temp-buffer From 02f7bf9a8392dec1ba142a0c77d966b93e020cdc Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Tue, 20 Aug 2019 18:41:29 -0300 Subject: [PATCH 3/9] Make CSV check for DataFrame - It now works with and without :session, for :results value and :results output --- ob-julia.el | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ob-julia.el b/ob-julia.el index 4dcbf4f..8bd43d2 100644 --- a/ob-julia.el +++ b/ob-julia.el @@ -81,9 +81,12 @@ This function is called by `org-babel-execute-src-block'." (or (equal "yes" rownames-p) (org-babel-pick-name (cdr (assq :rowname-names params)) rownames-p))))) - (if graphics-file nil (if (and result (sequencep result)) - (org-babel-normalize-newline result) - result))))) + (if graphics-file nil result)))) + +; Code from https://github.com/gjkerns/ob-julia/pull/14 +; (if graphics-file nil (if (and result (sequencep result)) +; (org-babel-normalize-newline result) +; result))))) (defun org-babel-normalize-newline (result) (replace-regexp-in-string @@ -201,9 +204,9 @@ end" (defvar org-babel-julia-eoe-output "org_babel_julia_eoe") (defvar org-babel-julia-write-object-command "begin - local p_ans = [ %s ] - using CSV - CSV.write(\"%s\", p_ans) + local p_ans = %s + using CSV, DataFrames + CSV.write(\"%s\", typeof(p_ans) <: DataFrame ? p_ans : DataFrame(:ans => p_ans)) p_ans end") @@ -227,7 +230,7 @@ last statement in BODY, as elisp." (let ((tmp-file (org-babel-temp-file "julia-"))) (org-babel-eval org-babel-julia-command (format org-babel-julia-write-object-command - (format "(function () %s end)()" body) + (format "begin %s end" body) (org-babel-process-file-name tmp-file 'noquote) )) (org-babel-julia-process-value-result From d7dbef69f4d3839ff8e1d4a11f0c3a107b013737 Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Thu, 20 May 2021 15:29:42 -0300 Subject: [PATCH 4/9] Use some defconst's to enable Emacs reloading values --- ob-julia.el | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ob-julia.el b/ob-julia.el index 8bd43d2..199519b 100644 --- a/ob-julia.el +++ b/ob-julia.el @@ -17,7 +17,7 @@ (declare-function ess-wait-for-process "ext:ess-inf" (&optional proc sec-prompt wait force-redisplay)) -(defconst org-babel-header-args:julia +(defvar org-babel-header-args:julia '((width . :any) (horizontal . :any) (results . ((file list vector table scalar verbatim) @@ -200,8 +200,8 @@ end" (and (member "graphics" (cdr (assq :result-params params))) (cdr (assq :file params)))) -(defvar org-babel-julia-eoe-indicator "print(\"org_babel_julia_eoe\")") -(defvar org-babel-julia-eoe-output "org_babel_julia_eoe") +(defconst org-babel-julia-eoe-indicator "print(\"org_babel_julia_eoe\")") +(defconst org-babel-julia-eoe-output "org_babel_julia_eoe") (defvar org-babel-julia-write-object-command "begin local p_ans = %s From 38405b4e5fd48f38a9ad5c4dc41dbf70046dbf39 Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Thu, 20 May 2021 15:30:18 -0300 Subject: [PATCH 5/9] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f8ad57e..7f63a6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .juliahistory *~ +/ob-julia.elc From 28eac9ddd9af6edf05ca592740550dad1d82842d Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Thu, 20 May 2021 15:30:26 -0300 Subject: [PATCH 6/9] Assume julia is installed and available --- ob-julia.el | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ob-julia.el b/ob-julia.el index 199519b..8a80060 100644 --- a/ob-julia.el +++ b/ob-julia.el @@ -30,9 +30,7 @@ (defvar org-babel-default-header-args:julia '()) -(defvar inferior-julia-program "julia") - -(defcustom org-babel-julia-command inferior-julia-program +(defcustom org-babel-julia-command "julia" "Name of command to use for executing julia code." :group 'org-babel :version "24.3" From 15dd2d185c4a4c307cc946e28779e3778d44daba Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Thu, 20 May 2021 15:30:57 -0300 Subject: [PATCH 7/9] Treat "nothing" results properly Add error-catching to detect future CSV writing problems --- ob-julia.el | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/ob-julia.el b/ob-julia.el index 8a80060..ef8254a 100644 --- a/ob-julia.el +++ b/ob-julia.el @@ -201,11 +201,37 @@ end" (defconst org-babel-julia-eoe-indicator "print(\"org_babel_julia_eoe\")") (defconst org-babel-julia-eoe-output "org_babel_julia_eoe") -(defvar org-babel-julia-write-object-command "begin +(defconst org-babel-julia-write-object-command "begin local p_ans = %s - using CSV, DataFrames - CSV.write(\"%s\", typeof(p_ans) <: DataFrame ? p_ans : DataFrame(:ans => p_ans)) - p_ans + local p_tmp_file = \"%s\" + + try + using CSV, DataFrames + + if typeof(p_ans) <: DataFrame + p_ans_df = p_ans + else + p_ans_df = DataFrame(:ans => p_ans) + end + + CSV.write(p_tmp_file, + p_ans_df, + writeheader = %s, + transform = (col, val) -> something(val, missing), + missingstring = \"nil\", + quotestrings = false) + p_ans + catch e + err_msg = \"Source block evaluation failed. $e\" + CSV.write(p_tmp_file, + DataFrame(:ans => err_msg), + writeheader = false, + transform = (col, val) -> something(val, missing), + missingstring = \"nil\", + quotestrings = false) + + err_msg + end end") (defun org-babel-julia-evaluate From 5995316695c88a0e871916f0f527cfd892b8bdcc Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Thu, 20 May 2021 15:32:34 -0300 Subject: [PATCH 8/9] Parse 'column-names-p' when writing results --- ob-julia.el | 1 + 1 file changed, 1 insertion(+) diff --git a/ob-julia.el b/ob-julia.el index ef8254a..2cb76c9 100644 --- a/ob-julia.el +++ b/ob-julia.el @@ -286,6 +286,7 @@ last statement in BODY, as elisp." (format org-babel-julia-write-object-command "ans" (org-babel-process-file-name tmp-file 'noquote) + (if column-names-p "true" "false") )) (org-babel-julia-process-value-result (org-babel-result-cond result-params From 6f85239b5ad78c31d63c1463b0e4f1dee43c0f75 Mon Sep 17 00:00:00 2001 From: Pedro Bruel Date: Thu, 20 May 2021 17:00:05 -0300 Subject: [PATCH 9/9] Update syntax for sessionless evaluations --- ob-julia.el | 1 + 1 file changed, 1 insertion(+) diff --git a/ob-julia.el b/ob-julia.el index 2cb76c9..dc33dba 100644 --- a/ob-julia.el +++ b/ob-julia.el @@ -256,6 +256,7 @@ last statement in BODY, as elisp." (format org-babel-julia-write-object-command (format "begin %s end" body) (org-babel-process-file-name tmp-file 'noquote) + (if column-names-p "true" "false") )) (org-babel-julia-process-value-result (org-babel-result-cond result-params