Skip to content

Commit

Permalink
FEAT: implemented transcode/one for translating just one value
Browse files Browse the repository at this point in the history 
related to: Oldes/Rebol-issues#1329
related to: Oldes/Rebol-issues#1916
  • Loading branch information
@Oldes
Oldes committed Aug 4, 2022
1 parent 769b5e5 commit fe44958
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 80 deletions.
1 change: 1 addition & 0 deletions src/boot/natives.reb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -655,6 +655,7 @@ transcode: native [
{Translates UTF-8 binary source to values. Returns one or several values in a block.}
source [binary! string!] "UTF-8 input buffer; string argument will be UTF-8 encoded"
/next "Translate next complete value (blocks as single value)"
/one "Translate next complete value (returns the value only)"
/only "Translate only a single value (blocks dissected)"
/error "Do not cause errors - return error object as value in place"
]
Expand Down
17 changes: 13 additions & 4 deletions src/core/l-scan.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1891,7 +1891,11 @@ extern REBSER *Scan_Full_Block(SCAN_STATE *scan_state, REBYTE mode_char);
***********************************************************************/
{
SCAN_STATE scan_state;
REBVAL *src = D_ARG(1);
REBVAL *src = D_ARG(1);
REBOOL next = D_REF(2);
REBOOL one = D_REF(3);
REBOOL only = D_REF(4);
REBOOL error = D_REF(5);
REBSER *blk;
REBSER *ser;
REBYTE *bin;
Expand All @@ -1910,12 +1914,17 @@ extern REBSER *Scan_Full_Block(SCAN_STATE *scan_state, REBYTE mode_char);

Init_Scan_State(&scan_state, bin, len);

if (D_REF(2)) SET_FLAG(scan_state.opts, SCAN_NEXT);
if (D_REF(3)) SET_FLAG(scan_state.opts, SCAN_ONLY);
if (D_REF(4)) SET_FLAG(scan_state.opts, SCAN_RELAX);
if (next || one) SET_FLAG(scan_state.opts, SCAN_NEXT);
if (only) SET_FLAG(scan_state.opts, SCAN_ONLY);
if (error) SET_FLAG(scan_state.opts, SCAN_RELAX);

blk = Scan_Code(&scan_state, 0);
DS_RELOAD(ds); // in case stack moved

if (one) {
*D_RET = *BLK_SKIP(blk, 0);
return IS_END(D_RET) ? R_UNSET : R_RET;
}
Set_Block(D_RET, blk);

if (scan_state.opts) {
Expand Down
163 changes: 87 additions & 76 deletions src/tests/units/lexer-test.r3
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@ Rebol [
--assert [[1 + 1] ""] = transcode/next "[1 + 1]"
--assert [[1 + 1] #{}] = transcode/next to binary! "[1 + 1]"

--test-- "transcode/one"
;@@ https://github.com/Oldes/Rebol-issues/issues/1329
--assert unset? transcode/one ""
--assert 1 = transcode/one "1 2"
--assert [1 2] = transcode/one "[1 2]"

===end-group===

===start-group=== "Invalid construction"
Expand Down Expand Up @@ -417,99 +423,104 @@ Rebol [
--assert error? try [load {#[file! "ab" 2 x]}]
--test-- {object!}
;@@ https://github.com/Oldes/Rebol-issues/issues/864
--assert block? try [transcode "#[object! [a: 1 b: 2]]"]
--assert block? try [transcode/only "#[object! [a: 1 b: 2]]"]
--assert block? try [transcode "#[object! [a: 1 b: 2]]"]
--assert block? try [transcode/only "#[object! [a: 1 b: 2]]"]
--assert object? try [transcode/one "#[object! [a: 1 b: 2]]"]
--test-- {function!}
;@@ https://github.com/Oldes/Rebol-issues/issues/1114
--assert function? first transcode {#[function! [[a [series!]][print a]]]}
--assert function? transcode/one {#[function! [[a [series!]][print a]]]}

--test-- {datatype!}
;@@ https://github.com/Oldes/Rebol-issues/issues/2508
--assert datatype? first transcode/only {#[unset!]}
--assert datatype? first transcode/only {#[none!]}
--assert datatype? first transcode/only {#[logic!]}
--assert datatype? first transcode/only {#[integer!]}
--assert datatype? first transcode/only {#[decimal!]}
--assert datatype? first transcode/only {#[percent!]}
--assert datatype? first transcode/only {#[money!]}
--assert datatype? first transcode/only {#[char!]}
--assert datatype? first transcode/only {#[pair!]}
--assert datatype? first transcode/only {#[tuple!]}
--assert datatype? first transcode/only {#[time!]}
--assert datatype? first transcode/only {#[date!]}
--assert datatype? first transcode/only {#[binary!]}
--assert datatype? first transcode/only {#[string!]}
--assert datatype? first transcode/only {#[file!]}
--assert datatype? first transcode/only {#[email!]}
--assert datatype? first transcode/only {#[ref!]}
--assert datatype? first transcode/only {#[url!]}
--assert datatype? first transcode/only {#[tag!]}
--assert datatype? first transcode/only {#[bitset!]}
--assert datatype? first transcode/only {#[image!]}
--assert datatype? first transcode/only {#[vector!]}
--assert datatype? first transcode/only {#[block!]}
--assert datatype? first transcode/only {#[paren!]}
--assert datatype? first transcode/only {#[path!]}
--assert datatype? first transcode/only {#[set-path!]}
--assert datatype? first transcode/only {#[get-path!]}
--assert datatype? first transcode/only {#[lit-path!]}
--assert datatype? first transcode/only {#[map!]}
--assert datatype? first transcode/only {#[datatype!]}
--assert datatype? first transcode/only {#[typeset!]}
--assert datatype? first transcode/only {#[word!]}
--assert datatype? first transcode/only {#[set-word!]}
--assert datatype? first transcode/only {#[get-word!]}
--assert datatype? first transcode/only {#[lit-word!]}
--assert datatype? first transcode/only {#[refinement!]}
--assert datatype? first transcode/only {#[issue!]}
--assert datatype? first transcode/only {#[native!]}
--assert datatype? first transcode/only {#[action!]}
--assert datatype? first transcode/only {#[rebcode!]}
--assert datatype? first transcode/only {#[command!]}
--assert datatype? first transcode/only {#[op!]}
--assert datatype? first transcode/only {#[closure!]}
--assert datatype? first transcode/only {#[function!]}
--assert datatype? first transcode/only {#[frame!]}
--assert datatype? first transcode/only {#[object!]}
--assert datatype? first transcode/only {#[module!]}
--assert datatype? first transcode/only {#[error!]}
--assert datatype? first transcode/only {#[task!]}
--assert datatype? first transcode/only {#[port!]}
--assert datatype? first transcode/only {#[gob!]}
--assert datatype? first transcode/only {#[event!]}
--assert datatype? first transcode/only {#[handle!]}
--assert datatype? first transcode/only {#[struct!]}
--assert datatype? first transcode/only {#[library!]}
--assert datatype? first transcode/only {#[utype!]}
--assert datatype? transcode/one {#[unset!]}
--assert datatype? transcode/one {#[none!]}
--assert datatype? transcode/one {#[logic!]}
--assert datatype? transcode/one {#[integer!]}
--assert datatype? transcode/one {#[decimal!]}
--assert datatype? transcode/one {#[percent!]}
--assert datatype? transcode/one {#[money!]}
--assert datatype? transcode/one {#[char!]}
--assert datatype? transcode/one {#[pair!]}
--assert datatype? transcode/one {#[tuple!]}
--assert datatype? transcode/one {#[time!]}
--assert datatype? transcode/one {#[date!]}
--assert datatype? transcode/one {#[binary!]}
--assert datatype? transcode/one {#[string!]}
--assert datatype? transcode/one {#[file!]}
--assert datatype? transcode/one {#[email!]}
--assert datatype? transcode/one {#[ref!]}
--assert datatype? transcode/one {#[url!]}
--assert datatype? transcode/one {#[tag!]}
--assert datatype? transcode/one {#[bitset!]}
--assert datatype? transcode/one {#[image!]}
--assert datatype? transcode/one {#[vector!]}
--assert datatype? transcode/one {#[block!]}
--assert datatype? transcode/one {#[paren!]}
--assert datatype? transcode/one {#[path!]}
--assert datatype? transcode/one {#[set-path!]}
--assert datatype? transcode/one {#[get-path!]}
--assert datatype? transcode/one {#[lit-path!]}
--assert datatype? transcode/one {#[map!]}
--assert datatype? transcode/one {#[datatype!]}
--assert datatype? transcode/one {#[typeset!]}
--assert datatype? transcode/one {#[word!]}
--assert datatype? transcode/one {#[set-word!]}
--assert datatype? transcode/one {#[get-word!]}
--assert datatype? transcode/one {#[lit-word!]}
--assert datatype? transcode/one {#[refinement!]}
--assert datatype? transcode/one {#[issue!]}
--assert datatype? transcode/one {#[native!]}
--assert datatype? transcode/one {#[action!]}
--assert datatype? transcode/one {#[rebcode!]}
--assert datatype? transcode/one {#[command!]}
--assert datatype? transcode/one {#[op!]}
--assert datatype? transcode/one {#[closure!]}
--assert datatype? transcode/one {#[function!]}
--assert datatype? transcode/one {#[frame!]}
--assert datatype? transcode/one {#[object!]}
--assert datatype? transcode/one {#[module!]}
--assert datatype? transcode/one {#[error!]}
--assert datatype? transcode/one {#[task!]}
--assert datatype? transcode/one {#[port!]}
--assert datatype? transcode/one {#[gob!]}
--assert datatype? transcode/one {#[event!]}
--assert datatype? transcode/one {#[handle!]}
--assert datatype? transcode/one {#[struct!]}
--assert datatype? transcode/one {#[library!]}
--assert datatype? transcode/one {#[utype!]}
--test-- {direct values}
--assert logic? first transcode/only {#[true]}
--assert logic? first transcode/only {#[false]}
--assert none? first transcode/only {#[none]}
--assert unset? first transcode/only {#[unset]}
--assert logic? transcode/one {#[true]}
--assert logic? transcode/one {#[false]}
--assert none? transcode/one {#[none]}
--assert unset? transcode/one {#[unset]}

===end-group===

===start-group=== "BINARY"
--test-- {binary! with spaces}
--assert #{00} = first transcode/only " #{0 0}"
--assert #{00} = first transcode/only "2#{0000 00 00}"
--assert #{00} = first transcode/only "2#{0000^/0000}"
--assert #{00} = first transcode/only "2#{0000^M0000}"
--assert #{01} = first transcode/only "2#{0000^-0001}"
--assert #{02} = first transcode/only "2#{0000^ 0010}"
--assert #{0001} = first transcode/only "16#{00 01}"
--assert #{0001} = first transcode/only "64#{AA E=}"
--assert #{00} = transcode/one " #{0 0}"
--assert #{00} = transcode/one "2#{0000 00 00}"
--assert #{00} = transcode/one "2#{0000^/0000}"
--assert #{00} = transcode/one "2#{0000^M0000}"
--assert #{01} = transcode/one "2#{0000^-0001}"
--assert #{02} = transcode/one "2#{0000^ 0010}"
--assert #{0001} = transcode/one "16#{00 01}"
--assert #{0001} = transcode/one "64#{AA E=}"

--test-- {binary! with comments inside}
;@@ https://github.com/Oldes/Rebol-wishes/issues/23
--assert #{00} = first transcode/only/error "#{;XXX^/00}"
--assert #{00} = first transcode/only/error "#{00;XXX^/}"
--assert #{0002} = first transcode/only/error "#{00;XXX^/02}"
--assert #{0002} = first transcode/only/error "#{00;XXX^M02}" ;CR is also comment stopper
--assert #{00} = transcode/one/error "#{;XXX^/00}"
--assert #{00} = transcode/one/error "#{00;XXX^/}"
--assert #{0002} = transcode/one/error "#{00;XXX^/02}"
--assert #{0002} = transcode/one/error "#{00;XXX^M02}" ;CR is also comment stopper

--assert error? transcode/one/error "#{0}"

--test-- {binary! with other valid escapes}
--assert #{0003} = first transcode/only/error "#{^(30)^(30)03}"
--assert #{0003} = transcode/one/error "#{^(30)^(30)03}"
--test-- {binary! with unicode char} ; is handled early
--assert error? first transcode/only/error "#{0č}"
--assert error? transcode/one/error "#{0č}"
--test-- "Invalid binary"
;@@ https://github.com/Oldes/Rebol-issues/issues/1431
--assert all [error? e: try [load {000016#{FF}}] e/id = 'invalid e/arg1 = "integer"]
Expand Down

0 comments on commit fe44958

Please sign in to comment.